{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 二十一点 Blackjack-v0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import matplotlib.pyplot as plt\n",
    "import gym"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 环境使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "观察空间 = Tuple(Discrete(32), Discrete(11), Discrete(2))\n",
      "动作空间 = Discrete(2)\n",
      "动作数量 = 2\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"Blackjack-v0\")\n",
    "env.seed(0)\n",
    "print('观察空间 = {}'.format(env.observation_space))\n",
    "print('动作空间 = {}'.format(env.action_space))\n",
    "print('动作数量 = {}'.format(env.action_space.n))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 在线回合更新\n",
    "回合更新预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ob2state(observation):\n",
    "    return observation[0], observation[1], int(observation[2])\n",
    "\n",
    "def evaluate_action_monte_carlo(env, policy, episode_num=500000):\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # 玩一回合\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # 回合结束\n",
    "        g = reward # 回报\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "    return q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = np.zeros((22, 11, 2, 2))\n",
    "policy[20:, :, :, 0] = 1 # >= 20 时收手\n",
    "policy[:20, :, :, 1] = 1 # < 20 时继续\n",
    "\n",
    "q = evaluate_action_monte_carlo(env, policy) # 动作价值\n",
    "v = (q * policy).sum(axis=-1) # 状态价值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(data):\n",
    "    fig, axes = plt.subplots(1, 2, figsize=(9, 4))\n",
    "    titles = ['without ace', 'with ace']\n",
    "    have_aces = [0, 1]\n",
    "    extent = [12, 22, 1, 11]\n",
    "    for title, have_ace, axis in zip(titles, have_aces, axes):\n",
    "        dat = data[extent[0]:extent[1], extent[2]:extent[3], have_ace].T\n",
    "        axis.imshow(dat, extent=extent, origin='lower')\n",
    "        axis.set_xlabel('player sum')\n",
    "        axis.set_ylabel('dealer showing')\n",
    "        axis.set_title(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "观测 = (11, 10, False)\n",
      "玩家 = [8, 3], 庄家 = [10, 9]\n",
      "动作 = 1\n",
      "观测 = (12, 10, False), 奖励 = 0, 结束指示 = False\n",
      "玩家 = [8, 3, 1], 庄家 = [10, 9]\n",
      "动作 = 0\n",
      "观测 = (12, 10, False), 奖励 = -1.0, 结束指示 = True\n",
      "随机策略 奖励：-1.0\n"
     ]
    }
   ],
   "source": [
    "def play_once(env, policy=None):\n",
    "    total_reward = 0\n",
    "    observation = env.reset()\n",
    "    print('观测 = {}'.format(observation))\n",
    "    while True:\n",
    "        print('玩家 = {}, 庄家 = {}'.format(env.player, env.dealer))\n",
    "        if policy is None:\n",
    "            action = np.random.choice(env.action_space.n)\n",
    "        print('动作 = {}'.format(action))\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        print('观测 = {}, 奖励 = {}, 结束指示 = {}'.format(\n",
    "                observation, reward, done))\n",
    "        total_reward += reward\n",
    "        if done:\n",
    "            return total_reward # 回合结束\n",
    "    \n",
    "print(\"随机策略 奖励：{}\".format(play_once(env)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHhlJREFUeJzt3XmYZAdZ7/Hvb3pmMksyGciCkAQGMEHAiyEOSAxrgAjIol4XvOReCEhceARxQUBE8eGKgKKozwVzbyQRckGWoCGghsWEwIVgGJIQiAjGQJYhK9nJTPf0e/+oM9iZXqa655w+XTXfz/P001Wnqt/zVk+d37x96lSdVBWSJEnLbVXfDUiSpP2TQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ8h+KMmdSR6ywO1XJXnacvYkaTyZN1qIQ8h+qKoOrKorAZKckeSNy7HeJC9K8pnlWJeklaGvvNFocAiRJEm9cAgZE0lOSfKRGde/keT9M65fneTY5nIl+f4kpwIvAF7V7DL9yIySxya5LMltSf42yboZtV7a1L8lyTlJHtAs39LUXj3jvucn+YUkDwfeCRzfrOvWBR7HFUnuSHJlkl/c4/bnJbkkye1J/j3JM5rlByc5Pcn2JNcmeWOSiX34lUqax3LmzR7rfWiSTyW5OclNSc5KsnnG7UclOTvJjc19/nLGbS9usuU7Sf4pyYNa/JVoiRxCxscFwBOSrEpyf2ANcAJA83rsgcBlM3+gqk4DzgLe0uwyfc6Mm38WeAbwYOBRwIuaWicCb2puvz/wTeB9e2uuqq4Afgn4XLOuzfPc9Qbg2cAm4BTgT5Mc16z7scDfAL8FbAaeCFzV/NyZwBTw/cCjgZOAX9hbX5KWZFnyZg5hkD8PAB4OHAX8frPeCeBcBpm0BTiCJpuS/ATwWuCngMOAC4H3LvGxq0Wr934XjYKqujLJHcCxwDHAPzH46+IHgOOBC6tqehEl/7yqrgNo/mI5tln+AuCvq2pbc9trgO8k2dLS4/jojKsXJDkPeAKwDXhJs+6PN7df2/RwP+CZwOaq+i5wV5I/BU4F/qqNviT9p2XMmz3X+w3gG83VG5O8Dfi95vpjGQwnv1VVU82y3ceg/SLwpuaPIZL8IfDaJA+qqm8uok+1zCFkvFwAPJnB3oALgFuBJzEIhQsWWevbMy7fzWDjpvm+bfcNVXVnkpsZ/NVx7ZK6niHJMxmEyjEM9tRtAL7c3HwU8LE5fuxBDP4S255k97JVwNX72o+keS1H3txLksOBP2fwh8lBDLbz7zQ3HwV8c8YAMtODgLcn+ZOZ5RjklkNIj3w5ZrzsDoUnNJcvYBAKT2L+UFjsaZSvY7BBA5BkI3AIgwHkrmbxhhn3/75h15XkAOBDwB8D92tesvkYg7CAwVDx0Dl+9GpgB3BoVW1uvjZV1SOHflSSFms58mZPb2pqPKqqNgEnc+98eODMY9JmuBr4xRn5sLmq1lfV/9vHfrSPHELGywXAU4D1VXUNg9c9n8FgSPjSPD9zPTDve/jn8H+BU5Ic2wwNfwhcVFVXVdWNDIaRk5NMJHkx9x4argeOTLJ2ntprgQOAG4GpZq/ISTNuP71Z91Ob16KPSPIDVbUdOA/4kySbmtsemuRJi3hckhZnOfJmTwcBdwK3JjmCwfFhu30B2A78UZKNSdYlOaG57Z3Aa5I8Er53IPvP7EMfaolDyBipqn9jsIFe2Fy/HbgS+GxV7Zrnx04HHpHk1iR/N8Q6Pgn8LoM9FtsZDBnPn3GXlzIIhpuBRwIz/9L4FPAV4NtJbpqj9h3Ay4H3M9jF+t+Ac2bc/gWag1WB2xiE4O69Mv+DwRDz1eZnP8jgwFlJHViOvJnDG4DjGGz/HwXOntHPLuA5DF4e+hZwDfBzzW0fBt4MvC/J7cDlDI4jU89Sta97xyRJkhbPPSGSJKkXDiGSJKkXDiGSJKkXDiGSJKkXI/FhZWtXb6j1aw5uv3BXB+V2Uber44cX9aGGi6nbTdmu/s06OUD76G42r4es+87e77RIV189xc23TGfv9xxda9dsrHXr5jtbwMqTXSP0poGusnS6q3zqqt/26049pJscOXr9za3XXEqOjMQQsn7NwRz/kFPaLzw51wfr7bvsmGy/aEcbY91zTyd12TXfO/T2TXX0b1ZTHdT9q0Pbrwm85+j37/1Oi3TSs2a9Y3rsrFu3mcc8+ldar5uO/j+buKODbbOj/9PTVZbe3VE+TXaQ0XSTp7f8+WGt1wQ497+c2XrNpeSIL8dIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqRedDaEJPnrJDckuXzGsvsm+XiSrzff79PV+iWNPnNEGm9d7gk5A3jGHsteDXyyqo4GPtlcl6T5nIE5Io2tzoaQqvo0cMsei58HnNlcPhP4ia7WL2n0mSPSeFu9zOu7X1VtB6iq7UkOn++OSU4FTgVYt/Zgpjesbb2ZTHX08JP2S+6cbL0mQNav76RuTXbU78REJ3X5brVe8hUP/ETrNQEOndjYes3Vs/6fX9GWlCMHrN/MjkPaz5F1N+1svSbAroPWtV5z1c5drdfs1PoDuql7wJpOymZN+3Vfe/THWq8JKydHVuyBqVV1WlVtraqta1Zv6LsdSSNoZo6sPqD90JW0b5Z7CLk+yf0Bmu83LPP6JY0+c0QaE8s9hJwDvLC5/ELg75d5/ZJGnzkijYku36L7XuBzwMOSXJPkJcAfAU9P8nXg6c11SZqTOSKNt84OTK2qn5/npqd2tU5J48Uckcbbij0wVZIkjTeHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1AuHEEmS1IvVfTcwtOn2S2ZyV/tFAZLWS9aabv6pMj3ZTd0OfgcANV2d1O3CfSfu7KTurppovWYxOr/XpaqJcM/m9n93sLaDmrDmjvbzaeLOHa3XBNi16YBO6k7c3klZsrOb3GN1+8+viXTwnx+wq9qvu5QccU+IJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqRS9DSJJXJvlKksuTvDfJuj76kDS6zBFp9C37EJLkCODlwNaq+kFgAnj+cvchaXSZI9J46OvlmNXA+iSrgQ3AdT31IWl0mSPSiFu93CusqmuT/DHwLeC7wHlVdd5ef3CUjl6ZaL/ZTO1qvSYAq9JJ2do13UldVnf0lJ2aar3kPbWm9ZoA03T0XBghS8mRiZ3Fpm/uaL2XXesmWq8JkOlqvebUQQe0XhNg9Z07O6nbmdXd/JvVdPu5N0H7zwOA6Y7qLlYfL8fcB3ge8GDgAcDGJCfPcb9Tk1yc5OLJqbuXu01JK9hScmTnzruWu01Je9HH/oWnAf9RVTdW1SRwNvCje96pqk6rqq1VtXXN6g3L3qSkFW3RObJ27cZlb1LSwvoYQr4FPC7JhiQBngpc0UMfkkaXOSKNgWUfQqrqIuCDwDbgy00Ppy13H5JGlzkijYdlPzAVoKp+D/i9PtYtaTyYI9LoG6X3nEiSpDHiECJJknqx15djkvzUHItvA75cVTe035KkcWOOSJrLMMeEvAQ4Hvjn5vqTgc8DxyT5g6p6d0e9SRof5oikWYYZQqaBh1fV9QBJ7ge8A/gR4NOA4SFpb8wRSbMMc0zIlt3B0bgBOKaqbgEmu2lL0pgxRyTNMsyekAuTnAt8oLn+X4FPJ9kI3NpZZ5LGiTkiaZZhhpCXMQiME4AAfwN8qKoKeEqHvUkaH+aIpFn2OoQ0IfHB5kuSFs0ckTSXvR4TkuSnknw9yW1Jbk9yR5Lbl6M5SePBHJE0l2FejnkL8Jyq8uRQkpbKHJE0yzDvjrne4JC0j8wRSbMMsyfk4iR/C/wdsGP3wqo6u7OuJI0bc0TSLMMMIZuAu4GTZiwrwPCQNCxzRNIsw7w75pTlaETS+DJHJM1l3iEkyauq6i1J/oLBXyz3UlUv77QzSSPPHJG0kIX2hOw+iOzi5WhkISnIjl3tF65ZmdiOyan2a05Pt18TqHt27P1OK8nkzm7q7mr/+XXd5H1arwlw99rrWq85PXs+aMuKyZEKTK2faL3umrs72N6B7Gx/m09XmddRPqWrfPrObZ2Uzdq1rdf8952Ht14T4PHrvt56zaXkyLxDSFV9pLl4YVVdudSmJO2/zBFJCxnmwNQzkhwB/AuDs11eWFVf7rYtSWPGHJE0yzAHpj4xyVrgMcCTgY8mObCq7tt1c5LGgzkiaS57HUKSPB54QvO1GTgXuLDjviSNEXNE0lyGeTnmAgYHlb0J+FhVdXRkoKQxZo5ImmWYIeQQBqfffiLw8iTTwOeq6nc77UzSODFHJM0yzDEhtya5EjgKOBL4UWBN141JGh/miKS5DHNMyL8DXwM+A7wTOMVdqZIWwxyRNJdhXo45uqq6+SQaSfsLc0TSLKuGuM8Dknw4yQ1Jrk/yoSRHdt6ZpHFijkiaZZgh5F3AOcADgCOAjzTLJGlY5oikWYYZQg6rqndV1VTzdQZwWMd9SRov5oikWYYZQm5KcnKSiebrZODmrhuTNFbMEUmzDDOEvBj4WeDbwHbgp5tlkjQsc0TSLMN8Tsi3gOcuQy+SxpQ5Imkuw3xOyGHAS4EtM+9fVf4VI2ko5oikuQzzOSF/z+BEU58AdnXbjqQxZY5ImmWYIWRDVf12551IGmfmiKRZhjkw9dwkz2pzpUk2J/lgkn9NckWS49usL2nFMUckzTLvnpAkdwAFBHhtkh3AZHO9qmrTPqz37cA/VtVPJ1kLbNiHWpJWKHNE0kLmHUKq6qAuVphkE4PTeb+oWc9OwBNZSWPIHJG0kGHeHXMCcElV3dV8wNBxwJ81b7lbiocANwLvSvJDwBeBV1TVXXus91TgVIB1azYN98LRYk1WB0Uhuzo4T1cXNQGmu/kddCZdPBGgdrV/rOTNuw5svSbAdLX/b9b1s2Al5MgB6zcP9r+0bHqim+fkmsnJ1muuuuOe1msC0NHvIJNTndStQ+7TTd1bbm295p271rVeE2BX51v9cIZ55rwDuLvZ0F8FfBN49z6sczWDAHpHVT0auAt49Z53qqrTqmprVW1du3rjPqxO0grQe46sWWuOSCvNMEPIVFUV8Dzg7VX1dmBfdrFeA1xTVRc11z/IIEwkjS9zRNIswwwhdyR5DXAy8NEkE8Capa6wqr4NXJ3kYc2ipwJfXWo9SSPBHJE0yzBDyM8BO4CXNBv+EcBb93G9vwqcleQy4FjgD/exnqSVzRyRNMsw5475NvC2Gde/BfzNvqy0qi4Btu5LDUmjwxyRNJduDmmWJEnaC4cQSZLUiwWHkCQTSd6zXM1IGj/miKT5LDiEVNUu4LDmI5EladHMEUnzGeYsulcBn01yDoMPBAKgqt42709I0r1dhTkiaQ/DDCHXNV+r2LcPF5K0/zJHJM0yzFt03wCQZOOe52WQpGGYI5Lmstd3xyQ5PslXgSua6z+U5H913pmksWGOSJrLMG/R/TPgx4CbAarqUgan0JakYZkjkmYZ6nNCqurqPRa1f95zSWPNHJG0p2EOTL06yY8C1bzF7uU0u1QlaUjmiKRZhtkT8kvAyxiccOoaBieKelmXTUkaO+aIpFmGeXfMTcALlqEXSWPKHJE0l3mHkCR/AdR8t1fVyzvpSNLYMEckLWShPSEXL1sXksaVOSJpXvMOIVV15nI2Imn8mCOSFrLXY0KSHAb8NvAIYN3u5VV1Yod9zbZqqHcTL85EBzWBWjPMm44WJ1MdvZtxVbqpOzXVSdnaubOTupmY6KSuBlZEjhRkuv2yqyY7KApUB9tmrVvTek2AXRu6OTfhxNr2sxRg1e13d1K37ru59ZqrclXrNQGma95XSZdsKRWH+V/4LAZvpXsw8AYGJ6L6lyWsS9L+yxyRNMswQ8ghVXU6MFlVF1TVi4HHddyXpPFijkiaZZh9XZPN9+1JfpzBmTCP7K4lSWPIHJE0yzBDyBuTHAz8BvAXwCbglZ12JWncmCOSZhnmw8rObS7eBjyl23YkjSNzRNJc9npMSJJjknwyyeXN9UcleV33rUkaF+aIpLkMc2Dq/wZeQ/OablVdBjy/y6YkjR1zRNIswwwhG6rqC3ss6+ZDICSNK3NE0izDDCE3JXkozeeQJPlpYHunXUkaN+aIpFmGeXfMy4DTgB9Ici3wH8DJnXYladyYI5JmGebdMVcCT0uyEVhVVXd035akcWKOSJrLvENIkl+fZzkAVfW2jnqSNCbMEUkLWWhPyEHN94cBjwHOaa4/B/h0l01JGhvmiKR5zTuEVNUbAJKcBxy3e/dpkt8HPrAs3UkaaeaIpIUM8+6YBwIzz5++E9jSSTeSxpU5ImmWYd4d827gC0k+zODtdT8JnNlpV5LGjTkiaZZh3h3zP5P8A/CEZtEpVfWlbtuSNE7MEUlzGWZPCFW1DdjW5oqTTAAXA9dW1bPbrC1p5TFHJO1pmGNCuvIK4Ioe1y9p9Jkj0gjrZQhJciTw48D/6WP9kkafOSKNvr72hPwZ8Cpguqf1Sxp95og04oY6JqRNSZ4N3FBVX0zy5AXudypwKsC6NZtguoOcqWq/Zpd1O5BV3cyhtWqik7qdaT7BcxTsYnSeX11ZUo6sPZi1t+5ov5epbmagu4/c0HrN9dd9t/WaABN3tf97BVh1212d1O1qe8/t7f9+Jzra3ldKjvSxJ+QE4LlJrgLeB5yY5D173qmqTquqrVW1de3qjcvdo6SVbdE5smaNOSKtNMs+hFTVa6rqyKraAjwf+FRVeTZNSUMzR6Tx0Oe7YyRJ0n5s2Y8JmamqzgfO77MHSaPNHJFGl3tCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCJElSL1b33cDQpjuouauLopCpXe0XnZxsvyZQVZ3UZWqqk7KZmOikbnXQ72R102sXz9qOngUrSq0KUxvXtF539V3dbJsHfvXm1mvufMDBrdcEmLjtu53UpaPtne/e00nZLvL0+slNrdeElZMj7gmRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9WPYhJMlRSf45yRVJvpLkFcvdg6TRZo5I42F1D+ucAn6jqrYlOQj4YpKPV9VXe+hF0mgyR6QxsOx7Qqpqe1Vtay7fAVwBHLHcfUgaXeaINB762BPyPUm2AI8GLprjtlOBUwHWTRzEqtvubL+Bycn2awLTd93dSd1OdPQ7qKpu6u7Y0UndLrz9wpM6qbvjhDWt17x56rOt11wuQ+fI6k0ccM1t7Tdw83farwlw+CGtl1y77Rut1wTYdfvtndRdtXFjJ3Wn77qrk7okrZf80CeOb70mwLqT2s/+G6fOX/TP9HZgapIDgQ8Bv1ZVs57BVXVaVW2tqq1rJ9Yvf4OSVrxF5cjqDcvfoKQF9TKEJFnDIDjOqqqz++hB0mgzR6TR18e7YwKcDlxRVW9b7vVLGn3miDQe+tgTcgLw34ETk1zSfD2rhz4kjS5zRBoDy35galV9Bmj/6B1J+w1zRBoPfmKqJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqRaqq7x72KskdwNf67mNIhwI39d3EIoxSv6PUK4xWvw+rqoP6bqJL5kinRqnfUeoVRqvfRefI6q46adnXqmpr300MI8nFo9IrjFa/o9QrjFa/SS7uu4dlYI50ZJT6HaVeYbT6XUqO+HKMJEnqhUOIJEnqxagMIaf13cAijFKvMFr9jlKvMFr9jlKvSzVKj3GUeoXR6neUeoXR6nfRvY7EgamSJGn8jMqeEEmSNGYcQiRJUi9W3BCS5K+T3JDk8hnL3prkX5NcluTDSTb32eNuc/U647bfTFJJDu2jt7nM12+SX03ytSRfSfKWvvqbaZ7nwbFJPp/kkiQXJ3lsnz3uluSoJP+c5Irmd/iKZvl9k3w8ydeb7/fpu1dYsN8VuZ0thTnSjVHKEDBHutRajlTVivoCnggcB1w+Y9lJwOrm8puBN/fd53y9NsuPAv4J+CZwaN997uV3+xTgE8ABzfXD++5zgV7PA57ZXH4WcH7ffTa93B84rrl8EPBvwCOAtwCvbpa/egU9b+frd0VuZy0+f1bk4xulHBmlDFmgX3Ok234XtZ2tuD0hVfVp4JY9lp1XVVPN1c8DRy57Y3OYq9fGnwKvAlbUUb/z9PvLwB9V1Y7mPjcse2NzmKfXAjY1lw8GrlvWpuZRVduraltz+Q7gCuAI4HnAmc3dzgR+op8O722+flfqdrYU5kg3RilDwBzpUls5suKGkCG8GPiHvpuYT5LnAtdW1aV99zKkY4AnJLkoyQVJHtN3Qwv4NeCtSa4G/hh4Tc/9zJJkC/Bo4CLgflW1HQYbLHB4f53NbY9+Z1rR21kLVvTjG7EcGaUMAXOkdfuSIyM1hCT5HWAKOKvvXuaSZAPwO8Dr++5lEVYD9wEeB/wW8P4k6belef0y8MqqOgp4JXB6z/3cS5IDgQ8Bv1ZVt/fdz97M1+9K38721Up/fCOYI6OUIWCOtGpfc2RkhpAkLwSeDbygmhebVqCHAg8GLk1yFYPdUNuSfF+vXS3sGuDsGvgCMM3ghEkr0QuBs5vLHwBWxAFlAEnWMNgQz6qq3T1en+T+ze33B1bMbup5+h2V7WzJRuTxjVqOjFKGgDnSmjZyZCSGkCTPAH4beG5V3d13P/Opqi9X1eFVtaWqtjDYOI+rqm/33NpC/g44ESDJMcBaVu4ZG68DntRcPhH4eo+9fE/zV9/pwBVV9bYZN53DIPBovv/9cvc2l/n6HZXtbKlG5fGNYI6MUoaAOdKK1nKk7yNs9/wC3gtsByYZbHwvAb4BXA1c0ny9s+8+5+t1j9uvYoUc1b7A73Yt8B7gcmAbcGLffS7Q6+OBLwKXMnjt8Yf77rPp9fEMDna7bMZz9FnAIcAnGYTcJ4H79t3rXvpdkdtZi8+fFfn4RilHRilDFujXHOm230VtZ35suyRJ6sVIvBwjSZLGj0OIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOISHJ+kq199yFpdJkjWgqHEHUuyUTfPUgabebIeHII2U8k2ZLkX5OcmeSyJB9szlGx5/3ekeTiJF9J8oZm2VOTfHjGfZ6e5Ozm8klJPpdkW5IPNOcRIMlVSV6f5DPAz+yxjp9JcnmSS5N8uln2oiR/OeM+5yZ5cnP5ziRvTvLFJJ9I8tjmr64rmxN9SVoG5oja5hCyf3kYcFpVPQq4HfiVOe7zO1W1FXgU8KQkjwI+BTw8yWHNfU4B3pXkUOB1wNOq6jjgYuDXZ9S6p6oeX1Xv22Mdrwd+rKp+CBhm498InF9VPwzcAbwReDrwk8AfDPHzktpjjqg1DiH7l6ur6rPN5fcw+NjdPf1skm3Al4BHAo+owcfqvhs4Oclm4HgGp2d+HPAI4LNJLmFwXoMHzaj1t/P08VngjCQvBYbZxboT+Mfm8peBC6pqsrm8ZYifl9Qec0StWd13A1pWe35G/72uJ3kw8JvAY6rqO0nOANY1N78L+AhwD/CBqppqTmD08ar6+XnWd9ecTVT9UpIfAX4cuCTJsQxO+TxzKF434/Jk/ef5BaaBHU2d6SQ+h6XlZY6oNe4J2b88MMnxzeWfBz6zx+2bGGzwtyW5H/DM3TdU1XUMzj75OuCMZvHngROSfD9Akg3NWTQXlOShVXVRVb2ewdk2j2Jwkq5jk6xKchQr6PTaku7FHFFrnP72L1cAL0zyVwzOyPiOmTdW1aVJvgR8BbiSwe7Omc4CDquqrzb3vzHJi4D3Jjmguc/rgH/bSx9vTXI0EAZnhby0Wf4fDHaN7j4bp6SVxxxRazyL7n4iyRbg3Kr6wX2o8ZfAl6rq9Lb6kjQ6zBG1zT0hGkqSLzLYxfobffciaTSZI9qTe0IkSVIvPDBVkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT14v8DaTBxywLuRbsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "带起始探索的回合更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_with_exploring_start(env, episode_num=500000):\n",
    "    policy = np.zeros((22, 11, 2, 2))\n",
    "    policy[:, :, :, 1] = 1.\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # 随机选择起始状态和起始动作\n",
    "        state = (np.random.randint(12, 22),\n",
    "                 np.random.randint(1, 11),\n",
    "                 np.random.randint(2))\n",
    "        action = np.random.randint(2)\n",
    "        # 玩一回合\n",
    "        env.reset()\n",
    "        if state[2]: # 有A\n",
    "            env.player = [1, state[0] - 11]\n",
    "        else: # 没有A\n",
    "            if state[0] == 21:\n",
    "                env.player = [10, 9, 2]\n",
    "            else:\n",
    "                env.player = [10, state[0] - 10]\n",
    "        env.dealer[0] = state[1]\n",
    "        state_actions = []\n",
    "        while True:\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # 回合结束\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "        g = reward # 回报\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = 0.\n",
    "            policy[state][a] = 1.\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_with_exploring_start(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGhJJREFUeJzt3Xm0pHV95/H3h24UQRDFlrA0tkFxy8GWtEZEXHDBXeOo0ZEZRSOa8QQ1xgU1JuY47sEYPaNhgoLKuCEqogkqyqKjkLbZbbcgytICoizCiCzf+aOeNpe7dXXfW/W7T/X7dc49t+p56j7P9/bp+pzPfeqpelJVSJIkjds2rQeQJElbJ0uIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCErIVSvKbJH84z/qLkzxunDNJmkzmjeZjCdkKVdWdq+oigCTHJHnbOPab5EVJvjWOfUlaGlrljfrBEiJJkpqwhEyIJIcm+dKU+z9J8pkp9y9Jsrq7XUnuneQw4AXA67pDpl+assnVSc5Lcm2STyfZbsq2Xtpt/1dJTkyye7d8Vbft5VMee2qSP09yf+DDwP7dvq6Z5/dYn+T6JBcledm09c9Ick6S65L8R5IndsvvkuToJBuSXJbkbUmWLeCfVNIcxpk30/a7d5JvJLk6yS+THJdk5ynrVyY5IclV3WM+OGXdi7ts+XWSk5PccxH/SbSFLCGT4zTgwCTbJNkN2BY4AKB7PfbOwHlTf6CqjgKOA97dHTJ92pTVzwWeCNwL2Bd4Ubetg4B3dOt3A34GfGpTw1XVeuDlwHe6fe08x0OvBJ4K7AQcCrwvyX7dvh8KfAx4LbAz8Ejg4u7njgVuAe4NPBh4AvDnm5pL0hYZS97MIgzyZ3fg/sBK4O+6/S4DTmKQSauAPeiyKckzgTcCzwJWAGcAn9zC312LaPmmH6I+qKqLklwPrAb2AU5m8NfF/YD9gTOq6rbN2OQ/VdXlAN1fLKu75S8APlJV67p1RwC/TrJqkX6PL0+5e1qSrwIHAuuAl3T7/lq3/rJuhl2BJwE7V9X/A25I8j7gMOCfF2MuSf9pjHkzfb8/AX7S3b0qyZHA33b3H8qgnLy2qm7plm08B+1lwDu6P4ZI8nbgjUnuWVU/24w5tcgsIZPlNODRDI4GnAZcAzyKQSictpnb+sWU2zcyeHLTfV+3cUVV/SbJ1Qz+6rhsi6aeIsmTGITKPgyO1G0PnN+tXgl8ZZYfuyeDv8Q2JNm4bBvgkoXOI2lO48ib20lyD+CfGPxhsiOD5/mvu9UrgZ9NKSBT3RN4f5J/mLo5BrllCWnIl2Mmy8ZQOLC7fRqDUHgUc4fC5l5G+XIGT2gAkuwA7MKggNzQLd5+yuP/YNh9Jbkj8DngvcCu3Us2X2EQFjAoFXvP8qOXADcBd6+qnbuvnarqgUP/VpI21zjyZrp3dNvYt6p2Ag7h9vmw19Rz0qa4BHjZlHzYuaruVFX/d4HzaIEsIZPlNOAxwJ2q6lIGr3s+kUFJOHuOn7kCmPM9/LP4P8ChSVZ3peHtwJlVdXFVXcWgjBySZFmSF3P70nAFsGeSO8yx7TsAdwSuAm7pjoo8Ycr6o7t9P7Z7LXqPJPerqg3AV4F/SLJTt27vJI/ajN9L0uYZR95MtyPwG+CaJHswOD9so7OADcA7k+yQZLskB3TrPgwckeSB8PsT2Z+zgDm0SCwhE6SqfsTgCXpGd/864CLg21V16xw/djTwgCTXJPnCEPs4BfgbBkcsNjAoGc+b8pCXMgiGq4EHAlP/0vgGcCHwiyS/nGXb1wOHA59hcIj1vwInTll/Ft3JqsC1DEJw41GZ/86gxHy/+9njGZw4K2kExpE3s3grsB+D5/+XgROmzHMr8DQGLw/9HLgU+LNu3eeBdwGfSnIdcAGD88jUWKoWenRMkiRp83kkRJIkNWEJkSRJTVhCJElSE5YQSZLURC8+rOzud1tWq1Zu23qMpn503vabftAW2GffG0ey3VHNq9H4LTfwu7opm35kf5kjGrWtPfe2JEd6UUJWrdyWs05e2XqMpg7efdZPMV6wk08+ZyTbHdW8Go0z65TWI4ycOaJR29pzb0tyxJdjJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITIyshST6S5MokF0xZdrckX0vy4+77XUe1f0n9Z45Ik22UR0KOAZ44bdkbgFOq6j7AKd19SZrLMZgj0sQaWQmpqtOBX01b/Azg2O72scAzR7V/Sf1njkiTbfmY97drVW0AqKoNSe4x1wOTHAYcBrDXHuMec+k5+fJzWo8gLRXmiDQhluyJqVV1VFWtqao1K3ZZ1nocST1kjkhL27hLyBVJdgPovl855v1L6j9zRJoQ4y4hJwIv7G6/EPjimPcvqf/MEWlCjPItup8EvgPcN8mlSV4CvBN4fJIfA4/v7kvSrMwRabKN7Eytqnr+HKseO6p9Spos5og02ZbsiamSJGmyWUIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDWxvPUAkiSN08G7r249gjoeCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTTQpIUleneTCJBck+WSS7VrMIam/zBGp/8ZeQpLsARwOrKmqPwKWAc8b9xyS+ssckSZDq5djlgN3SrIc2B64vNEckvrLHJF6buwlpKouA94L/BzYAFxbVV8d9xyS+ssckSZDi5dj7go8A7gXsDuwQ5JDZnncYUnWJll71dW3jntMSUuYOSJNhhYvxzwO+GlVXVVVNwMnAA+f/qCqOqqq1lTVmhW7LBv7kJKWNHNEmgAtSsjPgYcl2T5JgMcC6xvMIam/zBFpArQ4J+RM4HhgHXB+N8NR455DUn+ZI9JkWN5ip1X1t8Dftti3pMlgjkj95yemSpKkJiwhkiSpiU2+HJPkWbMsvhY4v6quXPyRJE0ac0TSbIY5J+QlwP7AN7v7jwa+C+yT5O+r6uMjmk3S5DBHJM0wTAm5Dbh/VV0BkGRX4EPAnwCnA4aHpE0xRyTNMMw5Ias2BkfnSmCfqvoVcPNoxpI0YcwRSTMMcyTkjCQnAZ/t7v8X4PQkOwDXjGwySZPEHJE0wzAl5BUMAuMAIMDHgM9VVQGPGeFskiaHOSJphk2WkC4kju++JGmzmSOSZrPJc0KSPCvJj5Ncm+S6JNcnuW4cw0maDOaIpNkM83LMu4GnVZUXh5K0pcwRSTMM8+6YKwwOSQtkjkiaYZgjIWuTfBr4AnDTxoVVdcLIppI0acwRSTMMU0J2Am4EnjBlWQGGh6RhmSOSZhjm3TGHjmMQSZPLHJE0mzlLSJLXVdW7k3yAwV8st1NVh490Mkm9Z45Ims98R0I2nkS2dhyDzOdH523Pwbuvbj2GpM23ZHJE0tIzZwmpqi91N8+oqovGNI+kCWKOSJrPMCemHpNkD+DfGVzt8oyqOn+0Y0maMOaIpBmGOTH1kUnuADwEeDTw5SR3rqq7jXo4SZPBHJE0m02WkCSPAA7svnYGTgLOGPFckiaIOSJpNsO8HHMag5PK3gF8pap+N9qRJE0gc0TSDMOUkF0YXH77kcDhSW4DvlNVfzPSySRNEnNE0gzDnBNyTZKLgJXAnsDDgW1HPZikyWGOSJrNMOeE/AfwQ+BbwIeBQz2UKmlzmCOSZjPMyzH3qarbRj6JpElmjkiaYZshHrN7ks8nuTLJFUk+l2TPkU8maZKYI5JmGKaEfBQ4Edgd2AP4UrdMkoZljkiaYZgSsqKqPlpVt3RfxwArRjyXpMlijkiaYZgS8sskhyRZ1n0dAlw96sEkTRRzRNIMw5SQFwPPBX4BbACe3S2TpGGZI5JmGOZzQn4OPH0Ms0iaUOaIpNkM8zkhK4CXAqumPr6q/CtG0lDMEUmzGeZzQr7I4EJTXwduHe04kiaUOSJphmFKyPZV9fqRTyJpkpkjkmYY5sTUk5I8eTF3mmTnJMcn+UGS9Un2X8ztS1pyzBFJM8x5JCTJ9UABAd6Y5Cbg5u5+VdVOC9jv+4F/q6pnJ7kDsP0CtiVpiTJHJM1nzhJSVTuOYodJdmJwOe8Xdfv5HeCFrKQJZI5Ims8mX45JckCSHbrbhyQ5MsleC9jnHwJXAR9NcnaSf9m4/Wn7PSzJ2iRrb+amBexOUmtLIUeuutrzYaWlZphzQj4E3JjkQcDrgJ8BH1/APpcD+wEfqqoHAzcAb5j+oKo6qqrWVNWabbnjAnYnaQloniMrdlm2gN1JGoVhSsgtVVXAM4D3V9X7gYUcYr0UuLSqzuzuH88gTCRNLnNE0gzDlJDrkxwBHAJ8OckyYNst3WFV/QK4JMl9u0WPBb6/pduT1AvmiKQZhikhfwbcBLyke+LvAbxngfv9S+C4JOcBq4G3L3B7kpY2c0TSDMNcO+YXwJFT7v8c+NhCdlpV5wBrFrINSf1hjkiazTBHQiRJkhadJUSSJDUxbwlJsizJJ8Y1jKTJY45Imsu8JaSqbgVWdB+JLEmbzRyRNJdhrqJ7MfDtJCcy+EAgAKrqyDl/QpJu72LMEUnTDFNCLu++tmFhHy4kaetljkiaYZi36L4VIMkOVXXDph4vSdOZI5JmM8wF7PZP8n1gfXf/QUn+18gnkzQxzBFJsxnmLbr/CBwMXA1QVecyuIS2JA3LHJE0w1CfE1JVl0xb5DWxJW0Wc0TSdMOcmHpJkocD1b3F7nC6Q6qSNCRzRNIMwxwJeTnwCgYXnLqUwYWiXjHKoSRNHHNE0gzDvDvml8ALxjCLpAlljkiazZwlJMkHgJprfVUdPpKJJE0Mc0TSfOY7ErJ2bFNImlTmiKQ5zVlCqurYcQ4iafKYI5Lms8lzQpKsAF4PPADYbuPyqjpohHPdzj773sjJJ5+z6Ns9ePfVi75NSTMthRyRtPQM8+6Y4xi8le5ewFsZXIjq30c4k6TJY45ImmGYErJLVR0N3FxVp1XVi4GHjXguSZPFHJE0wzAfVnZz931DkqcwuBLmnqMbSdIEMkckzTBMCXlbkrsArwE+AOwEvHqkU0maNOaIpBmG+bCyk7qb1wKPGe04kiaROSJpNps8JyTJPklOSXJBd3/fJG8e/WiSJoU5Imk2w5yY+r+BI+he062q84DnjXIoSRPHHJE0wzAlZPuqOmvasltGMYykiWWOSJphmBLyyyR7013/IcmzgQ0jnUrSpDFHJM0wzLtjXgEcBdwvyWXAT4FDRjqVpEljjkiaYZh3x1wEPC7JDsA2VXX96MeSNEnMEUmzmbOEJPmrOZYDUFVHjmgmSRPCHJE0n/mOhOzYfb8v8BDgxO7+04DTRzmUpIlhjkia05wlpKreCpDkq8B+Gw+fJvk74LNjmU5Sr5kjkuYzzLtj9gJ+N+X+74BVI5lG0qQyRyTNMMy7Yz4OnJXk8wzeXvenwLEjnUrSpDFHJM0wzLtj/meSfwUO7BYdWlVnj3YsSZPEHJE0m2GOhFBV64B1i7njJMuAtcBlVfXUxdy2pKXHHJE03TDnhIzKK4H1Dfcvqf/MEanHmpSQJHsCTwH+pcX+JfWfOSL1X6sjIf8IvA64rdH+JfWfOSL13NhLSJKnAldW1fc28bjDkqxNsvaqq28d03SS+sAckSZDiyMhBwBPT3Ix8CngoCSfmP6gqjqqqtZU1ZoVuywb94ySljZzRJoAYy8hVXVEVe1ZVauA5wHfqCqvpilpaOaINBlavjtGkiRtxYb6nJBRqapTgVNbziCp38wRqb88EiJJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKmJsZeQJCuTfDPJ+iQXJnnluGeQ1G/miDQZljfY5y3Aa6pqXZIdge8l+VpVfb/BLJL6yRyRJsDYj4RU1YaqWtfdvh5YD+wx7jkk9Zc5Ik2GFkdCfi/JKuDBwJmzrDsMOAxgrz1GM+bJl58zku0KDt59desRtJVonSPqn1Flv7m3+ZqdmJrkzsDngFdV1XXT11fVUVW1pqrWrNhl2fgHlLTkmSNSvzUpIUm2ZRAcx1XVCS1mkNRv5ojUfy3eHRPgaGB9VR057v1L6j9zRJoMLY6EHAD8N+CgJOd0X09uMIek/jJHpAkw9jO1qupbQMa9X0mTwxyRJoOfmCpJkpqwhEiSpCYsIZIkqQlLiCRJasISIkmSmrCESJKkJiwhkiSpCUuIJElqwhIiSZKasIRIkqQmLCGSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqQlLiCRJaiJV1XqGTUpyPfDD1nMM6e7AL1sPsRn6NG+fZoV+zXvfqtqx9RCjZI6MVJ/m7dOs0K95NztHlo9qkkX2w6pa03qIYSRZ25dZoV/z9mlW6Ne8Sda2nmEMzJER6dO8fZoV+jXvluSIL8dIkqQmLCGSJKmJvpSQo1oPsBn6NCv0a94+zQr9mrdPs26pPv2OfZoV+jVvn2aFfs272bP24sRUSZI0efpyJESSJE0YS4gkSWpiyZWQJB9JcmWSC6Yse0+SHyQ5L8nnk+zccsaNZpt1yrq/TlJJ7t5ittnMNW+Sv0zywyQXJnl3q/mmmuP/weok301yTpK1SR7acsaNkqxM8s0k67t/w1d2y++W5GtJftx9v2vrWWHeeZfk82xLmCOj0acMAXNklBYtR6pqSX0BjwT2Ay6YsuwJwPLu9ruAd7Wec65Zu+UrgZOBnwF3bz3nJv5tHwN8Hbhjd/8ereecZ9avAk/qbj8ZOLX1nN0suwH7dbd3BH4EPAB4N/CGbvkbltD/27nmXZLPs0X8/7Mkf78+5UifMmSeec2R0c67Wc+zJXckpKpOB341bdlXq+qW7u53gT3HPtgsZpu18z7gdcCSOut3jnn/AnhnVd3UPebKsQ82izlmLWCn7vZdgMvHOtQcqmpDVa3rbl8PrAf2AJ4BHNs97FjgmW0mvL255l2qz7MtYY6MRp8yBMyRUVqsHFlyJWQILwb+tfUQc0nydOCyqjq39SxD2gc4MMmZSU5L8pDWA83jVcB7klwCvBc4ovE8MyRZBTwYOBPYtao2wOAJC9yj3WSzmzbvVEv6ebYIlvTv17Mc6VOGgDmy6BaSI70qIUneBNwCHNd6ltkk2R54E/CW1rNshuXAXYGHAa8FPpMkbUea018Ar66qlcCrgaMbz3M7Se4MfA54VVVd13qeTZlr3qX+PFuopf779TBH+pQhYI4sqoXmSG9KSJIXAk8FXlDdi01L0N7AvYBzk1zM4DDUuiR/0HSq+V0KnFADZwG3Mbhg0lL0QuCE7vZngSVxQhlAkm0ZPBGPq6qNM16RZLdu/W7AkjlMPce8fXmebbGe/H59y5E+ZQiYI4tmMXKkFyUkyROB1wNPr6obW88zl6o6v6ruUVWrqmoVgyfnflX1i8ajzecLwEEASfYB7sDSvWLj5cCjutsHAT9uOMvvdX/1HQ2sr6ojp6w6kUHg0X3/4rhnm81c8/blebal+vL79TBH+pQhYI4sikXLkdZn2E7/Aj4JbABuZvDkewnwE+AS4Jzu68Ot55xr1mnrL2aJnNU+z7/tHYBPABcA64CDWs85z6yPAL4HnMvgtcc/bj1nN+sjGJzsdt6U/6NPBnYBTmEQcqcAd2s96ybmXZLPs0X8/7Mkf78+5UifMmSeec2R0c67Wc8zP7ZdkiQ10YuXYyRJ0uSxhEiSpCYsIZIkqQlLiCRJasISIkmSmrCEiCSnJlnTeg5J/WWOaEtYQjRySZa1nkFSv5kjk8kSspVIsirJD5Icm+S8JMd316iY/rgPJVmb5MIkb+2WPTbJ56c85vFJTuhuPyHJd5KsS/LZ7joCJLk4yVuSfAt4zrR9PCfJBUnOTXJ6t+xFST445TEnJXl0d/s3Sd6V5HtJvp7kod1fXRd1F/qSNAbmiBabJWTrcl/gqKraF7gO+B+zPOZNVbUG2Bd4VJJ9gW8A90+yonvMocBHk9wdeDPwuKraD1gL/NWUbf22qh5RVZ+ato+3AAdX1YOAYZ78OwCnVtUfA9cDbwMeD/wp8PdD/LykxWOOaNFYQrYul1TVt7vbn2DwsbvTPTfJOuBs4IHAA2rwsbofBw5JsjOwP4PLMz8MeADw7STnMLiuwT2nbOvTc8zxbeCYJC8FhjnE+jvg37rb5wOnVdXN3e1VQ/y8pMVjjmjRLG89gMZq+mf03+5+knsBfw08pKp+neQYYLtu9UeBLwG/BT5bVbd0FzD6WlU9f4793TDrEFUvT/InwFOAc5KsZnDJ56mleLspt2+u/7y+wG3ATd12bkvi/2FpvMwRLRqPhGxd9kqyf3f7+cC3pq3ficET/tokuwJP2riiqi5ncPXJNwPHdIu/CxyQ5N4ASbbvrqI5ryR7V9WZVfUWBlfbXMngIl2rk2yTZCVL6PLakm7HHNGisf1tXdYDL0zyzwyuyPihqSur6twkZwMXAhcxONw51XHAiqr6fvf4q5K8CPhkkjt2j3kz8KNNzPGeJPcBwuCqkOd2y3/K4NDoxqtxSlp6zBEtGq+iu5VIsgo4qar+aAHb+CBwdlUdvVhzSeoPc0SLzSMhGkqS7zE4xPqa1rNI6idzRNN5JESSJDXhiamSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqYn/D85HOZeMBEAcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHnRJREFUeJzt3XuYZAdZ5/Hvb7rnkplkMgNJMCSRQTARcEOIAxLC/SYgF3VRcckuBCTqsoJ4QVBE8WFFLqKoz4LZjSRCFhQIGi4KCBICy8UQkhAYEQyBXCZ3kkxuc+t3/6gz2unbVPec06er8v08Tz9Tdar6PW/11Pn126dO1UlVIUmStNxW9d2AJEm6Z3IIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIuQdKcluSH1jg9suTPHk5e5I0nswbLcQh5B6oqg6uqssAkpyZ5PXLsd4kL0zy2eVYl6SVoa+80WhwCJEkSb1wCBkTSU5N8qFp17+V5G+mXb8iyQnN5UrywCSnAc8HXtnsMv3QtJInJLkkyS1J/jrJumm1XtLUvynJuUnu2yzf0tSenHbfTyf5+SQPAt4BnNSs6+YFHse2JDuSXJbkF2bc/pwkFyW5Ncm/JXlas/zQJGck2Z7kqiSvTzJxAD9SSfNYzryZsd4HJPlUkhuT3JDk7CSbpt1+TJJzklzf3OfPp932oiZbvpfkY0nu1+KPREvkEDI+zgMek2RVkiOB1cDJAM3rsQcDl0z/hqo6HTgbeFOzy/RZ027+GeBpwP2B44EXNrWeCLyhuf1I4DvAe/fXXFVtA34R+Hyzrk3z3PU64JnARuBU4I+TnNis+xHAXwG/AWwCHgtc3nzfWcAe4IHAw4CnAj+/v74kLcmy5M0cwiB/7gs8CDgG+L1mvRPAhxlk0hbgKJpsSvITwG8BPwUcDpwPvGeJj10tmtz/XTQKquqyJDuAE4BjgY8x+Ovih4CTgPOramoRJf+0qq4GaP5iOaFZ/nzgL6vqwua2VwPfS7KlpcfxkWlXz0vyceAxwIXAi5t1f6K5/aqmh/sATwc2VdWdwO1J/hg4DfiLNvqS9B+WMW9mrvdbwLeaq9cneSvwu831RzAYTn6jqvY0y/Ydg/YLwBuaP4ZI8gfAbyW5X1V9ZxF9qmUOIePlPODxDPYGnAfcDDyOQSict8ha10y7fAeDjZvm3wv33VBVtyW5kcFfHVctqetpkjydQagcy2BP3Xrgq83NxwAfnePb7sfgL7HtSfYtWwVccaD9SJrXcuTN3SQ5AvhTBn+YHMJgO/9ec/MxwHemDSDT3Q94W5I/ml6OQW45hPTIl2PGy75QeExz+TwGofA45g+FxZ5G+WoGGzQASTYA92YwgNzeLF4/7f7fN+y6kqwFPgC8BbhP85LNRxmEBQyGigfM8a1XADuBw6pqU/O1saoeMvSjkrRYy5E3M72hqXF8VW0ETuHu+fD9049Jm+YK4Bem5cOmqjqoqv7fAfajA+QQMl7OA54AHFRVVzJ43fNpDIaEr8zzPdcC876Hfw7/Fzg1yQnN0PAHwBer6vKqup7BMHJKkokkL+LuQ8O1wNFJ1sxTew2wFrge2NPsFXnqtNvPaNb9pOa16KOS/FBVbQc+DvxRko3NbQ9I8rhFPC5Ji7MceTPTIcBtwM1JjmJwfNg+XwK2A3+YZEOSdUlObm57B/DqJA+Bfz+Q/acPoA+1xCFkjFTVvzLYQM9vrt8KXAZ8rqr2zvNtZwAPTnJzkr8dYh2fBH6HwR6L7QyGjOdNu8tLGATDjcBDgOl/aXwK+BpwTZIb5qi9A3gZ8DcMdrH+F+Dcabd/ieZgVeAWBiG4b6/Mf2MwxHy9+d73MzhwVlIHliNv5vA64EQG2/9HgHOm9bMXeBaDl4e+C1wJ/Gxz2weBNwLvTXIrcCmD48jUs1Qd6N4xSZKkxXNPiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6sVIfFjZmsn1ddCa+T7l+wBMLeYD/Rahi4N9p7o6gLijuhPdnLZl7/rV3dSd703DB2DVxrk+M+nAPfCgG1uvecUVe7jppqns/56ja/WaDbVu3ebW66azbbN92TPfm1YOUFc/go4yutZ1lSPt/12/d1M3P4PjNlzXes0rrty76BwZiSHkoDWbeORx7Z8GJLff1XpNgOzc3XrN2rWr9ZoA7OnmFyX36mBoBG49/vBu6t6v/aFp/ZPb38gBzv1PZ7Ve88eeMesd02Nn3brNbH3E/2i97sSdHW1DHVh97S3dFO7oXZZ12x2d1N1z7FGd1L11y5zn3TsgNz/n9v3faQk++si3t17zGUvIEV+OkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvehsCEnyl0muS3LptGX3SvKJJN9s/t3c1foljT5zRBpvXe4JORN42oxlrwI+WVU/CHyyuS5J8zkTc0QaW50NIVX1GeCmGYufA5zVXD4L+Imu1i9p9Jkj0nibXOb13aeqtgNU1fYkR8x3xySnAacBrF23iV33Xt96MxMHr229JsDELXe1XjO7drdeE4CpqW7Kbjyok7q7Du5mbr79YXe2XvMtx/5d6zUBjpjY0HrN1bN+z69oS8qRNes3c+cRq1tvpla1XxMgHWyaaw9d035RgKpOyk7evqeTuncd3k32X/vo9v/T3vawD7ReE+DIyYNbr7k631v096zYA1Or6vSq2lpVW1evbj90JY2/u+XIOnNEWmmWewi5NsmRAM2/1y3z+iWNPnNEGhPLPYScC7ygufwCoJv91ZLGmTkijYku36L7HuDzwHFJrkzyYuAPgack+SbwlOa6JM3JHJHGW2cHplbVz81z05O6Wqek8WKOSONtxR6YKkmSxptDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6sVk3w0MZRXsXdf+vJSpar0mAIesbb3kxI7WS3aqJrqZb7/3oE7K8qwHf7X1mpsm7mi9JsDemmi9ZtHRtrCC7F0NO45u/3m55tZufnaTd7Zf867N7T93ANbesreTujvvvaaTurcd1c3P4aTjt7Vec+Oqu1qvCbC72v8/W0qOuCdEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1wiFEkiT1opchJMkrknwtyaVJ3pNkXR99SBpd5og0+pZ9CElyFPAyYGtV/TAwATxvufuQNLrMEWk89PVyzCRwUJJJYD1wdU99SBpd5og04iaXe4VVdVWStwDfBe4EPl5VH1/oe6Ymws5DJ1rvZWpNWq8JsHpH+3Vrspt5ceK2XZ3Uvf5HDu6k7hEnXNtJ3dXZ23rN3dX+c1YDS8mRmoA9HTwtU93kyJ4N7ddctbObXrt6qu9e302/t598eyd1J1e1nyN31erWawKsYk8ndRerj5djNgPPAe4P3BfYkOSUOe53WpILklyw565unjCSRtNScmTv7eaItNL08XLMk4FvV9X1VbUbOAd41Mw7VdXpVbW1qrZOruvgTwJJo2zROTKxwRyRVpo+hpDvAo9Msj5JgCcB23roQ9LoMkekMbDsQ0hVfRF4P3Ah8NWmh9OXuw9Jo8sckcbDsh+YClBVvwv8bh/rljQezBFp9PmJqZIkqRcOIZIkqRf7fTkmyU/NsfgW4KtVdV37LUkaN+aIpLkMc0zIi4GTgH9qrj8e+AJwbJLfr6p3ddSbpPFhjkiaZZghZAp4UFVdC5DkPsDbgR8FPgMYHpL2xxyRNMswx4Rs2RccjeuAY6vqJmB3N21JGjPmiKRZhtkTcn6SDwPva67/Z+AzSTYAN3fWmaRxYo5ImmWYIeSlDALjZCDAXwEfqKoCntBhb5LGhzkiaZb9DiFNSLy/+ZKkRTNHJM1lv8eEJPmpJN9MckuSW5PsSHLrcjQnaTyYI5LmMszLMW8CnlVVnhxK0lKZI5JmGebdMdcaHJIOkDkiaZZh9oRckOSvgb8Fdu5bWFXndNaVpHFjjkiaZZghZCNwB/DUacsKMDwkDcsckTTLMO+OOXU5GpE0vswRSXOZdwhJ8sqqelOSP2PwF8vdVNXLOu1M0sgzRyQtZKE9IfsOIrtgORpZSK2C3evTet3d6ydarwmQzR3Ubf/hA1AT6zqpu3NTNw3feuOhndS94pDNrdf85vrva70mwMPWXNl6zVnTQXtWTI4AVAdPy71r268JsGpP+zWnVrdfE2DP+mHe47B4nfV7Uze592+bD2u95uWHtl8T4Na132y95t4lJMm8Q0hVfai5eH5VXbbUpiTdc5kjkhYyzIGpZyY5CvhnBme7PL+qvtptW5LGjDkiaZZhDkx9bJI1wMOBxwMfSXJwVd2r6+YkjQdzRNJc9juEJHk08JjmaxPwYeD8jvuSNEbMEUlzGeblmPMYHFT2BuCjVbWr25YkjSFzRNIswwwh92Zw+u3HAi9LMgV8vqp+p9POJI0Tc0TSLMMcE3JzksuAY4CjgUcBHb1xStI4MkckzWWYY0L+DfgG8FngHcCp7kqVtBjmiKS5DPNyzA9W1VTnnUgaZ+aIpFmG+Zi7+yb5YJLrklyb5ANJju68M0njxByRNMswQ8g7gXOB+wJHAR9qlknSsMwRSbMMM4QcXlXvrKo9zdeZwOEd9yVpvJgjkmYZZgi5IckpSSaar1OAG7tuTNJYMUckzTLMEPIi4GeAa4DtwHObZZI0LHNE0izDfE7Id4FnL0MvksaUOSJpLsN8TsjhwEuALdPvX1X+FSNpKOaIpLkM8zkhf8fgRFP/COztth1JY8ockTTLMEPI+qr6zc47kTTOzBFJswxzYOqHkzyjzZUm2ZTk/Un+Jcm2JCe1WV/SimOOSJpl3j0hSXYABQT4rSQ7gd3N9aqqjQew3rcB/1BVz02yBlh/ALUkrVDmiKSFzDuEVNUhXawwyUYGp/N+YbOeXYAnspLGkDkiaSHDvDvmZOCiqrq9+YChE4E/ad5ytxQ/AFwPvDPJQ4EvAy+vqttnrPc04DSANRs2k1ri2hbSRU0gHZymq5PHD1RHhwhO3tFN3TtvXtNJ3St3bGq95k2HHtx6TYCdtaf1mlNdbQyNlZAjk4du7mRDmrwzrdcEqA7KpqPtfVX7T0kA9nZ0ysO11010UveGw9rf5r+9s5sPFr7r4G+0XrNq8dvXMMeEvB24o9nQXwl8B3jXotf0HyYZBNDbq+phwO3Aq2beqapOr6qtVbV18qANB7A6SStA7zkyscEckVaaYYaQPTUYb54DvK2q3gYcyC7WK4Erq+qLzfX3MwgTSePLHJE0yzBDyI4krwZOAT6SZAJYvdQVVtU1wBVJjmsWPQn4+lLrSRoJ5oikWYYZQn4W2Am8uNnwjwLefIDr/WXg7CSXACcAf3CA9SStbOaIpFmGOXfMNcBbp13/LvBXB7LSqroI2HogNSSNDnNE0lyG2RMiSZLUOocQSZLUiwWHkCQTSd69XM1IGj/miKT5LDiEVNVe4PDmI5EladHMEUnzGeYsupcDn0tyLoMPBAKgqt4673dI0t1djjkiaYZhhpCrm69VHNiHC0m65zJHJM0yzFt0XweQZMPM8zJI0jDMEUlz2e+7Y5KclOTrwLbm+kOT/K/OO5M0NswRSXMZ5i26fwL8GHAjQFVdzOAU2pI0LHNE0ixDfU5IVV0xY1FHJ4SWNK7MEUkzDXNg6hVJHgVU8xa7l9HsUpWkIZkjkmYZZk/ILwIvZXDCqSsZnCjqpV02JWnsmCOSZhnm3TE3AM9fhl4kjSlzRNJc5h1CkvwZUPPdXlUv66QjSWPDHJG0kIX2hFywbF1IGlfmiKR5zTuEVNVZy9mIpPFjjkhayH6PCUlyOPCbwIOBdfuWV9UTO+zr7gpW7W6/7OTOefcSH5hqv+7eNWm9JsDkrm5+BrsPmeik7vw79lde2bumVndQdTSthBzJFEzc2c121IW1N7f/rNxzUDePf+3NU53UnVrdTb9713ZTt4ufwlR10+tEOqi7hJrDvDvmbAZvpbs/8DoGJ6L650WvSdI9mTkiaZZhhpB7V9UZwO6qOq+qXgQ8suO+JI0Xc0TSLMN8WNm+F0K2J/lxBmfCPLq7liSNIXNE0izDDCGvT3Io8GvAnwEbgVd02pWkcWOOSJplmA8r+3Bz8RbgCd22I2kcmSOS5rLfY0KSHJvkk0kuba4fn+Q13bcmaVyYI5LmMsyBqf8beDXNa7pVdQnwvC6bkjR2zBFJswwzhKyvqi/NWLani2YkjS1zRNIswwwhNyR5AM3nOSV5LrC9064kjRtzRNIsw7w75qXA6cAPJbkK+DZwSqddSRo35oikWYZ5d8xlwJOTbABWVdWO7tuSNE7MEUlzmXcISfKr8ywHoKre2lFPksaEOSJpIQvtCTmk+fc44OHAuc31ZwGf6bIpSWPDHJE0r3mHkKp6HUCSjwMn7tt9muT3gPctS3eSRpo5Imkhw7w75vuBXdOu7wK2dNKNpHFljkiaZZh3x7wL+FKSDzJ4e91PAmd12pWkcWOOSJplmHfH/M8kfw88pll0alV9pdu2JI0Tc0TSXIbZE0JVXQhc2OaKk0wAFwBXVdUz26wtaeUxRyTNNMwxIV15ObCtx/VLGn3miDTCehlCkhwN/Djwf/pYv6TRZ45Io6+vPSF/ArwSmOpp/ZJGnzkijbihjglpU5JnAtdV1ZeTPH6B+50GnAawZv0mVt/Rfs5M7KrWawJMTab1mqluel21u5u6Xf1ayO72f7YAU9V+3dWrujlJ7G7a/z/r6FnQmaXkyOpDNrP6tvZ7WXNrNz+9XRvbf05Wny/AL8HU6r47WJxVq9p/LqzO3tZrAuzt4nfKEmr28ZQ8GXh2ksuB9wJPTPLumXeqqtOramtVbV299uDl7lHSyrboHJk8aMNy9yhpP5Z9CKmqV1fV0VW1BXge8Kmq8myakoZmjkjjYcR2zkmSpHGx7MeETFdVnwY+3WcPkkabOSKNLveESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXkz23cAwsreYvGOq/brVekkAVu1uv3BNtF6yKdxN2dW3d1N4Ymc6qbtrT1c/YK0YBdnbftmp1e3XhG56XXtzRxt8Rybv7KbfyTu7yZHbdrb/K3Xn1Ej8ml4y94RIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReLPsQkuSYJP+UZFuSryV5+XL3IGm0mSPSeJjsYZ17gF+rqguTHAJ8OcknqurrPfQiaTSZI9IYWPY9IVW1vaoubC7vALYBRy13H5JGlzkijYc+9oT8uyRbgIcBX5zjttOA0wDWTRzChkuuan39tWt36zUB2L2r9ZJTd97Vek2A2rmzk7prO6kKmx710E7qXv/dw1qv+fbjntJ6TYDdT2h/s71hz2dbr7lchs6RyUO47993kCMT3fwtl9vuaL3m3htubL1ml2rPnk7q3mvjxk7q3rnt2NZr/sOJj2y9JsDe57b/vL1mz8cW/T29HZia5GDgA8CvVNWtM2+vqtOramtVbV2z6qDlb1DSire4HFm//A1KWlAvQ0iS1QyC4+yqOqePHiSNNnNEGn19vDsmwBnAtqp663KvX9LoM0ek8dDHnpCTgf8KPDHJRc3XM3roQ9LoMkekMbDsB6ZW1WeBLPd6JY0Pc0QaD35iqiRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6kWqqu8e9ivJDuAbffcxpMOAG/puYhFGqd9R6hVGq9/jquqQvpvokjnSqVHqd5R6hdHqd9E5MtlVJy37RlVt7buJYSS5YFR6hdHqd5R6hdHqN8kFffewDMyRjoxSv6PUK4xWv0vJEV+OkSRJvXAIkSRJvRiVIeT0vhtYhFHqFUar31HqFUar31HqdalG6TGOUq8wWv2OUq8wWv0uuteRODBVkiSNn1HZEyJJksaMQ4gkSerFihtCkvxlkuuSXDpt2ZuT/EuSS5J8MMmmPnvcZ65ep93260kqyWF99DaX+fpN8stJvpHka0ne1Fd/083zPDghyReSXJTkgiSP6LPHfZIck+SfkmxrfoYvb5bfK8knknyz+Xdz373Cgv2uyO1sKcyRboxShoA50qXWcqSqVtQX8FjgRODSacueCkw2l98IvLHvPufrtVl+DPAx4DvAYX33uZ+f7ROAfwTWNteP6LvPBXr9OPD05vIzgE/33WfTy5HAic3lQ4B/BR4MvAl4VbP8VSvoeTtfvytyO2vx+bMiH98o5cgoZcgC/Zoj3fa7qO1sxe0JqarPADfNWPbxqtrTXP0CcPSyNzaHuXpt/DHwSmBFHfU7T7+/BPxhVe1s7nPdsjc2h3l6LWBjc/lQ4OplbWoeVbW9qi5sLu8AtgFHAc8BzmrudhbwE/10eHfz9btSt7OlMEe6MUoZAuZIl9rKkRU3hAzhRcDf993EfJI8G7iqqi7uu5chHQs8JskXk5yX5OF9N7SAXwHenOQK4C3Aq3vuZ5YkW4CHAV8E7lNV22GwwQJH9NfZ3Gb0O92K3s5asKIf34jlyChlCJgjrTuQHBmpISTJbwN7gLP77mUuSdYDvw28tu9eFmES2Aw8EvgN4G+SpN+W5vVLwCuq6hjgFcAZPfdzN0kOBj4A/EpV3dp3P/szX78rfTs7UCv98Y1gjoxShoA50qoDzZGRGUKSvAB4JvD8al5sWoEeANwfuDjJ5Qx2Q12Y5Pt67WphVwLn1MCXgCkGJ0xaiV4AnNNcfh+wIg4oA0iymsGGeHZV7evx2iRHNrcfCayY3dTz9Dsq29mSjcjjG7UcGaUMAXOkNW3kyEgMIUmeBvwm8OyquqPvfuZTVV+tqiOqaktVbWGwcZ5YVdf03NpC/hZ4IkCSY4E1rNwzNl4NPK65/ETgmz328u+av/rOALZV1Vun3XQug8Cj+ffvlru3uczX76hsZ0s1Ko9vBHNklDIEzJFWtJYjfR9hO/MLeA+wHdjNYON7MfAt4ArgoubrHX33OV+vM26/nBVyVPsCP9s1wLuBS4ELgSf23ecCvT4a+DJwMYPXHn+k7z6bXh/N4GC3S6Y9R58B3Bv4JIOQ+yRwr7573U+/K3I7a/H5syIf3yjlyChlyAL9miPd9ruo7cyPbZckSb0YiZdjJEnS+HEIkSRJvXAIkSRJvXAIkSRJvXAIkSRJvXAIEUk+nWRr331IGl3miJbCIUSdSzLRdw+SRps5Mp4cQu4hkmxJ8i9JzkpySZL3N+eomHm/tye5IMnXkryuWfakJB+cdp+nJDmnufzUJJ9PcmGS9zXnESDJ5Ulem+SzwE/PWMdPJ7k0ycVJPtMse2GSP592nw8neXxz+bYkb0zy5ST/mOQRzV9dlzUn+pK0DMwRtc0h5J7lOOD0qjoeuBX473Pc57eraitwPPC4JMcDnwIelOTw5j6nAu9MchjwGuDJVXUicAHwq9Nq3VVVj66q985Yx2uBH6uqhwLDbPwbgE9X1Y8AO4DXA08BfhL4/SG+X1J7zBG1xiHknuWKqvpcc/ndDD52d6afSXIh8BXgIcCDa/Cxuu8CTkmyCTiJwemZHwk8GPhckosYnNfgftNq/fU8fXwOODPJS4BhdrHuAv6hufxV4Lyq2t1c3jLE90tqjzmi1kz23YCW1czP6L/b9ST3B34deHhVfS/JmcC65uZ3Ah8C7gLeV1V7mhMYfaKqfm6e9d0+ZxNVv5jkR4EfBy5KcgKDUz5PH4rXTbu8u/7j/AJTwM6mzlQSn8PS8jJH1Br3hNyzfH+Sk5rLPwd8dsbtGxls8LckuQ/w9H03VNXVDM4++RrgzGbxF4CTkzwQIMn65iyaC0rygKr6YlW9lsHZNo9hcJKuE5KsSnIMK+j02pLuxhxRa5z+7lm2AS9I8hcMzsj49uk3VtXFSb4CfA24jMHuzunOBg6vqq83978+yQuB9yRZ29znNcC/7qePNyf5QSAMzgp5cbP82wx2je47G6eklcccUWs8i+49RJItwIer6ocPoMafA1+pqjPa6kvS6DBH1Db3hGgoSb7MYBfrr/Xdi6TRZI5oJveESJKkXnhgqiRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6sX/B0gLXkZaSzVLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "基于柔性策略的回合更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_with_soft(env, episode_num=500000, epsilon=0.1):\n",
    "    policy = np.ones((22, 11, 2, 2)) * 0.5 # 柔性策略\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # 玩一回合\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n, p=policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # 回合结束\n",
    "        g = reward # 回报\n",
    "        for state, action in state_actions:\n",
    "            c[state][action] += 1.\n",
    "            q[state][action] += (g - q[state][action]) / c[state][action]\n",
    "            # 更新策略为柔性策略\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = epsilon / 2.\n",
    "            policy[state][a] += (1. - epsilon)\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_with_soft(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGl5JREFUeJzt3Xm4XHWd5/H3h4TFIIhCpEkIxkZx6wcjHW0RccEFd21HbR2ZUbRFe3watW0X1NamH8e9UVuf0WYGBZVxQ1REu4OiLDoKHWLYjFsjCiSsyiK2rN/5o07sy91SSW7V71bl/Xqe+6TqnLrnfG5IffjeU6fqpKqQJEkatu1aB5AkSdsmhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hGyDkvw2yR/Psv7SJE8cZiZJ48m+0WwcQrZBVXX3qroEIMnxSd45jP0meWmS7w5jX5Lmh1Z9o9HgECJJkppwCBkTSQ5P8rUJ93+e5AsT7l+WZEV3u5LcL8kRwIuBN3aHTL82YZMrklyQ5IYkn0+y04RtvaLb/q+TnJJkSbd8ebfthRMee0aSv0zyIODjwIHdvq6f5edYl+SmJJckeeWk9c9OsjbJjUn+PclTuuX3SHJckg1JrkjyziQLtuKvVNIMhtk3k/a7b5JvJ7kuybVJTkyy24T1y5KcnOSa7jEfnbDuZV23/CbJqiT3mcO/Em0hh5DxcSZwcJLtkuwFbA8cBNC9Hnt34IKJ31BVxwInAu/rDpk+c8LqFwBPAe4L7A+8tNvWIcC7u/V7Ab8EPrepcFW1DngV8P1uX7vN8NCrgWcAuwKHAx9MckC370cAnwLeAOwGPAa4tPu+E4DbgfsBDwOeDPzlpnJJ2iJD6ZtphF7/LAEeBCwD/r7b7wLgVHqdtBxYStdNSZ4DvAV4LrAYOBv47Bb+7JpDCzf9EI2CqrokyU3ACmA/YBW93y4eCBwInF1Vd27GJv+pqtYDdL+xrOiWvxj4RFWt6dYdBfwmyfI5+jm+PuHumUlOAw4G1gAv7/b9zW79FV2GPYGnArtV1X8ANyf5IHAE8M9zkUvSfxpi30ze78+Bn3d3r0lyDPCO7v4j6A0nb6iq27tlG89BeyXw7u6XIZK8C3hLkvtU1S83I6fmmEPIeDkTeBy9owFnAtcDj6VXCmdu5raunHD7d/Se3HR/rtm4oqp+m+Q6er91XLFFqSdI8lR6pbIfvSN1i4ALu9XLgG9M8233ofeb2IYkG5dtB1y2tXkkzWgYfXMXSe4N/BO9X0x2ofc8/023ehnwywkDyET3AT6c5B8nbo5ebzmENOTLMeNlYykc3N0+k14pPJaZS2FzL6O8nt4TGoAkOwO70xtAbu4WL5rw+D/qd19JdgS+BHwA2LN7yeYb9MoCekPFvtN862XALcAeVbVb97VrVT2k759K0uYaRt9M9u5uG/tX1a7AYdy1H/aZeE7aBJcBr5zQD7tV1d2q6v9tZR5tJYeQ8XIm8HjgblV1Ob3XPZ9Cb0j44QzfcxUw43v4p/F/gcOTrOiGhncB51TVpVV1Db1h5LAkC5K8jLsODVcBeyfZYYZt7wDsCFwD3N4dFXnyhPXHdft+Qvda9NIkD6yqDcBpwD8m2bVbt2+Sx27GzyVp8wyjbybbBfgtcH2SpfTOD9voXGAD8J4kOyfZKclB3bqPA0cleQj84UT2529FDs0Rh5AxUlU/pfcEPbu7fyNwCfC9qrpjhm87DnhwkuuTfKWPfZwO/B29IxYb6A0ZL5zwkFfQK4brgIcAE3/T+DZwMXBlkmun2fZNwJHAF+gdYv2vwCkT1p9Ld7IqcAO9Etx4VOa/0xtiftR970n0TpyVNADD6JtpHA0cQO/5/3Xg5Al57gCeSe/loV8BlwN/0a37MvBe4HNJbgQuoncemRpL1dYeHZMkSdp8HgmRJElNOIRIkqQmHEIkSVITDiGSJKmJkfiwsj3utaCWL9u+dYymfnrBok0/SNpCv+dmbq1bsulHji57RIO2rff0lvTISAwhy5dtz7mrlrWO0dShS6b9FGNpTpxTp7eOMHD2iAZtW+/pLekRX46RJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTQxsCEnyiSRXJ7lowrJ7Jflmkp91f95zUPuXNPrsEWm8DfJIyPHAUyYtezNwelXdHzi9uy9JMzkee0QaWwMbQqrqLODXkxY/Gzihu30C8JxB7V/S6LNHpPG2cMj727OqNgBU1YYk957pgUmOAI4A2GfpsGPOP6vWrx3Idg9dsmIg25UGyB7RVrH35o95e2JqVR1bVSurauXi3Re0jiNpBNkj0vw27CHkqiR7AXR/Xj3k/UsaffaINCaGPYScAryku/0S4KtD3r+k0WePSGNikG/R/SzwfeABSS5P8nLgPcCTkvwMeFJ3X5KmZY9I421gZ2pV1YtmWPWEQe1T0nixR6TxNm9PTJUkSePNIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUxMLWAdSfQ5esGMh2V61fO5DtDiqvJGl8eCREkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNdFkCEnyuiQXJ7koyWeT7NQih6TRZY9Io2/oQ0iSpcCRwMqq+hNgAfDCYeeQNLrsEWk8tHo5ZiFwtyQLgUXA+kY5JI0ue0QacQuHvcOquiLJB4BfAf8BnFZVpw07x6hZtX5t6wjSvGGPSOOhxcsx9wSeDdwXWALsnOSwaR53RJLVSVZfc90dw44paR6zR6Tx0OLlmCcCv6iqa6rqNuBk4FGTH1RVx1bVyqpauXj3BUMPKWles0ekMdBiCPkV8Mgki5IEeAKwrkEOSaPLHpHGwNCHkKo6BzgJWANc2GU4dtg5JI0ue0QaD0M/MRWgqt4BvKPFviWNB3tEGn1+YqokSWrCIUSSJDWxyZdjkjx3msU3ABdW1dVzH0nSuLFHJE2nn3NCXg4cCHynu/844AfAfkn+oao+PaBsksaHPSJpin6GkDuBB1XVVQBJ9gQ+BvwZcBZgeUjaFHtE0hT9nBOyfGNxdK4G9quqXwO3DSaWpDFjj0iaop8jIWcnORX4Ynf/vwBnJdkZuH5gySSNE3tE0hT9DCGvplcYBwEBPgV8qaoKePwAs0kaH/aIpCk2OYR0JXFS9yVJm80ekTSdTZ4TkuS5SX6W5IYkNya5KcmNwwgnaTzYI5Km08/LMe8DnllVXhxK0payRyRN0c+7Y66yOCRtJXtE0hT9HAlZneTzwFeAWzYurKqTB5ZK0rixRyRN0c8QsivwO+DJE5YVYHlI6pc9ImmKft4dc/gwgkgaX/aIpOnMOIQkeWNVvS/JR+j9xnIXVXXkQJNJGnn2iKTZzHYkZONJZKuHEaSFQ5esaB1BGnfzpkd+esGigTznV61fO+fbVI8dPf5mHEKq6mvdzbOr6pIh5ZE0RuwRSbPp58TU45MsBf6N3tUuz66qCwcbS9KYsUckTdHPiamPSbID8HDgccDXk9y9qu416HCSxoM9Imk6mxxCkjwaOLj72g04FTh7wLkkjRF7RNJ0+nk55kx6J5W9G/hGVd062EiSxpA9ImmKfoaQ3eldfvsxwJFJ7gS+X1V/N9BkksaJPSJpin7OCbk+ySXAMmBv4FHA9oMOJml82COSptPPOSH/DvwE+C7wceBwD6VK2hz2iKTp9PNyzP2r6s6BJ5E0zuwRSVNs18djliT5cpKrk1yV5EtJ9h54MknjxB6RNEU/Q8gngVOAJcBS4GvdMknqlz0iaYp+hpDFVfXJqrq9+zoeWDzgXJLGiz0iaYp+hpBrkxyWZEH3dRhw3aCDSRor9oikKfoZQl4GvAC4EtgAPK9bJkn9skckTdHP54T8CnjWELJIGlP2iKTp9PM5IYuBVwDLJz6+qvwtRlJf7BFJ0+nnc0K+Su9CU98C7hhsHEljyh6RNEU/Q8iiqnrTwJNIGmf2iKQp+jkx9dQkT5vLnSbZLclJSX6cZF2SA+dy+5LmHXtE0hQzHglJchNQQIC3JLkFuK27X1W161bs98PAv1bV85LsACzaim1JmqfsEUmzmXEIqapdBrHDJLvSu5z3S7v93Ap4IStpDNkjkmbTz7tjDgLWVtXN3QcMHQB8qHvL3Zb4Y+Aa4JNJHgqcB7ymqm6etN8jgCMAdmIRhy5ZsYW7k9TafOiRfZYuZNXqtVv8M2j4Vq0frf9e/n9q8/VzTsjHgN91T/Q3Ar8EPr0V+1xIr4A+VlUPA24G3jz5QVV1bFWtrKqV27PjVuxO0jzQvEcW775gK3YnaRD6GUJur6oCng18uKo+DGzNIdbLgcur6pzu/kn0ykTS+LJHJE3RzxByU5KjgMOArydZAGy/pTusqiuBy5I8oFv0BOBHW7o9SSPBHpE0RT9DyF8AtwAv7574S4H3b+V+/xo4MckFwArgXVu5PUnzmz0iaYp+rh1zJXDMhPu/Aj61NTutqrXAyq3ZhqTRYY9Imk4/R0IkSZLmnEOIJElqYtYhJMmCJJ8ZVhhJ48cekTSTWYeQqroDWNx9JLIkbTZ7RNJM+rmK7qXA95KcQu8DgQCoqmNm/A5JuqtLsUckTdLPELK++9qOrftwIUnbLntE0hT9vEX3aIAkO0++LoMk9cMekTSdTb47JsmBSX4ErOvuPzTJ/xp4Mkljwx6RNJ1+3qL7IeBQ4DqAqjqf3iW0Jalf9oikKfr6nJCqumzSojsGkEXSGLNHJE3Wz4mplyV5FFDdW+yOpDukKkl9skckTdHPkZBXAa+md8Gpy+ldKOrVgwwlaezYI5Km6OfdMdcCLx5CFkljyh6RNJ0Zh5AkHwFqpvVVdeRAEkkaG/aIpNnMdiRk9dBSSBpX9oikGc04hFTVCcMMImn82COSZrPJc0KSLAbeBDwY2Gnj8qo6ZIC5JI2Rce6RQ5esGMh2V61fO5DtanD/zbT5+nl3zIn03kp3X+Boehei+rcBZpI0fuwRSVP0M4TsXlXHAbdV1ZlV9TLgkQPOJWm82COSpujnw8pu6/7ckOTp9K6EuffgIkkaQ/aIpCn6GULemeQewOuBjwC7Aq8baCpJ48YekTRFPx9Wdmp38wbg8YONI2kc2SOSprPJc0KS7Jfk9CQXdff3T/K2wUeTNC7sEUnT6efE1P8NHEX3mm5VXQC8cJChJI0de0TSFP0MIYuq6txJy24fRBhJY8sekTRFP0PItUn2pbv+Q5LnARsGmkrSuLFHJE3Rz7tjXg0cCzwwyRXAL4DDBppK0rixRyRN0c+7Yy4BnphkZ2C7qrpp8LEkjRN7RNJ0ZhxCkvzNDMsBqKpjBpRJ0piwRyTNZrYjIbt0fz4AeDhwSnf/mcBZgwwlaWzYI5JmNOMQUlVHAyQ5DThg4+HTJH8PfHEo6SSNNHtE0mz6eXfMPsCtE+7fCiwfSBpJ48oekTRFP++O+TRwbpIv03t73Z8DJww0laRxY49ImqKfd8f8zyT/AhzcLTq8qn442FiSxok9Imk6/RwJoarWAGvmcsdJFgCrgSuq6hlzuW1J8489Immyfs4JGZTXAOsa7l/S6LNHpBHWZAhJsjfwdOD/tNi/pNFnj0ijr9WRkA8BbwTubLR/SaPPHpFGXF/nhMylJM8Arq6q85I8bpbHHQEcAbATi4aUTtIo2JIe2WfpYOpu1fq1A9mutC1ocSTkIOBZSS4FPgcckuQzkx9UVcdW1cqqWrk9Ow47o6T5bbN7ZPHuC4adUdImDH0IqaqjqmrvqloOvBD4dlV5NU1JfbNHpPHQ8t0xkiRpGzb0c0ImqqozgDNaZpA02uwRaXR5JESSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqYmHrAP3Yb//fsWrV2jnf7qFLVsz5NiVJUn88EiJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJasIhRJIkNeEQIkmSmnAIkSRJTTiESJKkJhxCJElSEw4hkiSpCYcQSZLUhEOIJElqwiFEkiQ14RAiSZKacAiRJElNDH0ISbIsyXeSrEtycZLXDDuDpNFmj0jjYWGDfd4OvL6q1iTZBTgvyTer6kcNskgaTfaINAaGfiSkqjZU1Zru9k3AOmDpsHNIGl32iDQeWhwJ+YMky4GHAedMs+4I4AiAfZYOJuaq9WsHsl0NzqFLVrSOoHmmdY9I2nLNTkxNcnfgS8Brq+rGyeur6tiqWllVKxfvvmD4ASXNe/aINNqaDCFJtqdXHCdW1cktMkgabfaINPpavDsmwHHAuqo6Ztj7lzT67BFpPLQ4EnIQ8N+AQ5Ks7b6e1iCHpNFlj0hjYOhnalXVd4EMe7+Sxoc9Io0HPzFVkiQ14RAiSZKacAiRJElNOIRIkqQmHEIkSVITDiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECJJkppwCJEkSU04hEiSpCYcQiRJUhMOIZIkqQmHEEmS1IRDiCRJaiJV1TrDJiW5CfhJ6xx92gO4tnWIzTBKeUcpK4xW3gdU1S6tQwySPTJQo5R3lLLCaOXd7B5ZOKgkc+wnVbWydYh+JFk9KllhtPKOUlYYrbxJVrfOMAT2yICMUt5RygqjlXdLesSXYyRJUhMOIZIkqYlRGUKObR1gM4xSVhitvKOUFUYr7yhl3VKj9DOOUlYYrbyjlBVGK+9mZx2JE1MlSdL4GZUjIZIkacw4hEiSpCbm3RCS5BNJrk5y0YRl70/y4yQXJPlykt1aZtxouqwT1v1tkkqyR4ts05kpb5K/TvKTJBcneV+rfBPN8O9gRZIfJFmbZHWSR7TMuFGSZUm+k2Rd93f4mm75vZJ8M8nPuj/v2TorzJp3Xj7PtoQ9Mhij1CFgjwzSnPVIVc2rL+AxwAHARROWPRlY2N1+L/De1jlnytotXwasAn4J7NE65yb+bh8PfAvYsbt/79Y5Z8l6GvDU7vbTgDNa5+yy7AUc0N3eBfgp8GDgfcCbu+Vvnkf/bmfKOy+fZ3P472de/nyj1COj1CGz5LVHBpt3s55n8+5ISFWdBfx60rLTqur27u4PgL2HHmwa02XtfBB4IzCvzvqdIe9fAe+pqlu6x1w99GDTmCFrAbt2t+8BrB9qqBlU1YaqWtPdvglYBywFng2c0D3sBOA5bRLe1Ux55+vzbEvYI4MxSh0C9sggzVWPzLshpA8vA/6ldYiZJHkWcEVVnd86S5/2Aw5Ock6SM5M8vHWgWbwWeH+Sy4APAEc1zjNFkuXAw4BzgD2ragP0nrDAvdslm96kvBPN6+fZHJjXP9+I9cgodQjYI3Nua3pkpIaQJG8FbgdObJ1lOkkWAW8F3t46y2ZYCNwTeCTwBuALSdI20oz+CnhdVS0DXgcc1zjPXSS5O/Al4LVVdWPrPJsyU975/jzbWvP95xvBHhmlDgF7ZE5tbY+MzBCS5CXAM4AXV/di0zy0L3Bf4Pwkl9I7DLUmyR81TTW7y4GTq+dc4E56F0yaj14CnNzd/iIwL04oA0iyPb0n4olVtTHjVUn26tbvBcybw9Qz5B2V59kWG5Gfb9R6ZJQ6BOyROTMXPTISQ0iSpwBvAp5VVb9rnWcmVXVhVd27qpZX1XJ6T84DqurKxtFm8xXgEIAk+wE7MH+v2LgeeGx3+xDgZw2z/EH3W99xwLqqOmbCqlPoFR7dn18ddrbpzJR3VJ5nW2pUfr4R7JFR6hCwR+bEnPVI6zNsJ38BnwU2ALfRe/K9HPg5cBmwtvv6eOucM2WdtP5S5slZ7bP83e4AfAa4CFgDHNI65yxZHw2cB5xP77XHP22ds8v6aHonu10w4d/o04DdgdPpldzpwL1aZ91E3nn5PJvDfz/z8ucbpR4ZpQ6ZJa89Mti8m/U882PbJUlSEyPxcowkSRo/DiGSJKkJhxBJktSEQ4gkSWrCIUSSJDXhECKSnJFkZesckkaXPaIt4RCigUuyoHUGSaPNHhlPDiHbiCTLk/w4yQlJLkhyUneNismP+1iS1UkuTnJ0t+wJSb484TFPSnJyd/vJSb6fZE2SL3bXESDJpUnenuS7wPMn7eP5SS5Kcn6Ss7plL03y0QmPOTXJ47rbv03y3iTnJflWkkd0v3Vd0l3oS9IQ2COaaw4h25YHAMdW1f7AjcD/mOYxb62qlcD+wGOT7A98G3hQksXdYw4HPplkD+BtwBOr6gBgNfA3E7b1+6p6dFV9btI+3g4cWlUPBfp58u8MnFFVfwrcBLwTeBLw58A/9PH9kuaOPaI54xCybbmsqr7X3f4MvY/dnewFSdYAPwQeAjy4eh+r+2ngsCS7AQfSuzzzI4EHA99LspbedQ3uM2Fbn58hx/eA45O8AujnEOutwL92ty8Ezqyq27rby/v4fklzxx7RnFnYOoCGavJn9N/lfpL7An8LPLyqfpPkeGCnbvUnga8Bvwe+WFW3dxcw+mZVvWiG/d08bYiqVyX5M+DpwNokK+hd8nniULzThNu31X9eX+BO4JZuO3cm8d+wNFz2iOaMR0K2LfskObC7/SLgu5PW70rvCX9Dkj2Bp25cUVXr6V198m3A8d3iHwAHJbkfQJJF3VU0Z5Vk36o6p6reTu9qm8voXaRrRZLtkixjHl1eW9Jd2COaM05/25Z1wEuS/DO9KzJ+bOLKqjo/yQ+Bi4FL6B3unOhEYHFV/ah7/DVJXgp8NsmO3WPeBvx0Eznen+T+QOhdFfL8bvkv6B0a3Xg1Tknzjz2iOeNVdLcRSZYDp1bVn2zFNj4K/LCqjpurXJJGhz2iueaREPUlyXn0DrG+vnUWSaPJHtFkHgmRJElNeGKqJElqwiFEkiQ14RAiSZKacAiRJElNOIRIkqQm/j8vKFFGwo1sLwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHqlJREFUeJzt3XmYZQV55/Hvr6v3pptGBKKAtqLgNoqkNSKuuESNS5LJoiMzbpEk40RjFqOJY2IeJ0ZNTIx5RsMMEaKMGhUjoom4IhqXIAKixGgQRWh22aG3euePezoWtfWt6nPq1L18P89TT9177q33vFV9z6/fe+6596SqkCRJWmor+m5AkiTdNTmESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiE3AUluSXJfee5/dIkT17KniSNJ/NG83EIuQuqqv2q6hKAJKckecNSrDfJC5N8YSnWJWl56CtvNBocQiRJUi8cQsZEkhcl+eiU699N8vdTrl+W5OjmciW5X5ITgecDr2p2mX50Ssmjk1yY5MYk70+ydkqtlzb1r09yRpJ7Nsu3NLVXTrnv55L8SpIHAu8Ejm3WdcM8v8fFSW5OckmSX512+3OSnJ/kpiT/nuRpzfL9k5ycZFuSy5O8IcnEPvxJJc1hKfNm2nqPSPKZJNcluTbJaUk2T7n98CSnJ7mmuc9fT7ntxU22/CjJJ5Lcu8U/iRbJIWR8nA08NsmKJPcAVgHHATSvx+4HXDj1B6rqJOA04M3NLtNnTbn5l4CnAfcBHgq8sKl1PPDG5vZ7AN8H3re35qrqYuDXgC8169o8x12vBp4JbAJeBPxFkmOadT8S+Dvgd4HNwOOAS5ufOxXYBdwPeDjwVOBX9taXpEVZkryZRRjkzz2BBwKHA3/UrHcCOJNBJm0BDqXJpiQ/C/w+8PPAQcA5wHsX+burRSv3fheNgqq6JMnNwNHAkcAnGDy7eABwLHBOVU0uoORfVdUVAM0zlqOb5c8H/raqzmtuew3woyRbWvo9Pjbl6tlJzgIeC5wHvKRZ9yeb2y9vejgEeDqwuapuB25N8hfAicDftNGXpB9bwryZvt7vAt9trl6T5K3AHzbXH8lgOPndqtrVLNtzDNqvAm9sngyR5E+A309y76r6/gL6VMscQsbL2cATGOwNOBu4AXg8g1A4e4G1rpxy+TYGGzfN9/P23FBVtyS5jsGzjssX1fUUSZ7OIFSOZLCnbj3wjebmw4GPz/Jj92bwTGxbkj3LVgCX7Ws/kua0FHlzJ0kOBv6KwROTjQy28x81Nx8OfH/KADLVvYG3JfnzqeUY5JZDSI98OWa87AmFxzaXz2YQCo9n7lBY6GmUr2CwQQOQZANwIIMB5NZm8fop9/+JYdeVZA3wIeDPgEOal2w+ziAsYDBUHDHLj14GbAfuXlWbm69NVfXgoX8rSQu1FHkz3RubGg+tqk3ACdw5H+419Zi0KS4DfnVKPmyuqnVV9c/72I/2kUPIeDkbeCKwrqp+yOB1z6cxGBK+PsfPXAXM+R7+Wfw/4EVJjm6Ghj8BvlJVl1bVNQyGkROSTCR5MXceGq4CDkuyeo7aq4E1wDXArmavyFOn3H5ys+4nNa9FH5rkAVW1DTgL+PMkm5rbjkjy+AX8XpIWZinyZrqNwC3ADUkOZXB82B5fBbYBf5pkQ5K1SY5rbnsn8JokD4b/OJD9F/ehD7XEIWSMVNW/MdhAz2mu3wRcAnyxqnbP8WMnAw9KckOSfxhiHZ8G/ieDPRbbGAwZz51yl5cyCIbrgAcDU59pfAb4JnBlkmtnqX0z8HLg7xnsYv0vwBlTbv8qzcGqwI0MQnDPXpn/xmCI+Vbzsx9kcOCspA4sRd7M4vXAMQy2/48Bp0/pZzfwLAYvD/0A+CHwy81tHwbeBLwvyU3ARQyOI1PPUrWve8ckSZIWzj0hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFyPxYWWrV66vdavn+pTvfTDZ0UG5XRzsO7mQDx9cBlZ189Davbab08HsXtN+zZUbd7ZfFLjfuutbr/mDy3Zx3fWT2fs9R9eq1Rtq7doDWq+brg7u76JsR72moyytFaP1kNy9rv18mtw81xuN9s2RG2a8QXGfXXbZLq5fYI6MxBCybvVmHnVU+6cBye07Wq8JkB3t/+dTt9/Res0u1UF366TujQ/uYBgFbrxv+zsFD3riFa3XBPjIg9o/5cXxT7+69ZrLzdq1B7D1kf+j9boT27v5T6KLJ0krdsz2YaIt1L29m4F7cs2qTup25fqHbWq95m3Pvqn1mgCfeMRJrdf86WcsfLDx5RhJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktSLzoaQJH+b5OokF01Zdrckn0zyneb7AV2tX9LoM0ek8dblnpBTgKdNW/Zq4NNVdX/g0811SZrLKZgj0tjqbAipqs8D109b/Bzg1ObyqcDPdrV+SaPPHJHG28olXt8hVbUNoKq2JTl4rjsmORE4EWDN2s3sOHB9682surGbGSzbV7Vfc/3a1msC5PbtndSdXDXRTd2OHrHb/9Ntrdd89X3/sfWaAPuvWNd6zYnROjxsUTmyev0B3H5w+9vmqls6eqyvTus1V+yo1msCZHdHdbspSya7KXzNo3a3XvOND/546zUBDp7Y0HrNVTOeL+zdsk2eqjqpqrZW1dZVq9r/Y0kaf3fKkbXmiLTcLPUQclWSewA0369e4vVLGn3miDQmlnoIOQN4QXP5BcBHlnj9kkafOSKNiS7fovte4EvAUUl+mOQlwJ8CT0nyHeApzXVJmpU5Io23zg5MrarnzXHTk7pap6TxYo5I423ZHpgqSZLGm0OIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqhUOIJEnqxcq+GxhKYHJVWi+7c/+1rdcEWLFjd+s1V1/+o9ZrAtTqVZ3U3b3f6k7qXvew9h8HAM856sLWa65fsb31mgA7q/3HV1Gt11x2qvlq2XUP6SZG11/ZfrOrb5lsvSZAJrvZLnet6abujUd08/z7sQ+7qPWaW1Zd23pNgN3V/t92MTninhBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktQLhxBJktSLXoaQJK9M8s0kFyV5b5K1ffQhaXSZI9LoW/IhJMmhwMuBrVX1EGACeO5S9yFpdJkj0njo6+WYlcC6JCuB9cAVPfUhaXSZI9KIW7nUK6yqy5P8GfAD4HbgrKo6a96fWRm2H9B+qyvvmGy9JsCqm9qvufMem9svCqzYsbuTutccva6TuhsfcF0ndQ9cdWvrNXfWROs1B7r5Nxsli8mR3Wvgxvu2/7xrzQ3Vek2A2w9K6zV3r+7meeeude33CrBzYydl4egOQhqYSPuPhRsm17deE2AXt7ReczG/fR8vxxwAPAe4D3BPYEOSE2a534lJzk1y7s472v8PQtLoWkyO7LrNHJGWmz5ejnky8L2quqaqdgKnA4+efqeqOqmqtlbV1lVrNyx5k5KWtQXnyMr15oi03PQxhPwAeFSS9UkCPAm4uIc+JI0uc0QaA0s+hFTVV4APAucB32h6OGmp+5A0uswRaTws+YGpAFX1h8Af9rFuSePBHJFGn5+YKkmSeuEQIkmSerHXl2OS/Pwsi28EvlFVV7ffkqRxY45Ims0wx4S8BDgW+Gxz/QnAl4Ejk/xxVb27o94kjQ9zRNIMwwwhk8ADq+oqgCSHAO8Afgr4PGB4SNobc0TSDMMcE7JlT3A0rgaOrKrrgZ3dtCVpzJgjkmYYZk/IOUnOBD7QXP/PwOeTbABu6KwzSePEHJE0wzBDyMsYBMZxQIC/Az5UVQU8scPeJI0Pc0TSDHsdQpqQ+GDzJUkLZo5Ims1ejwlJ8vNJvpPkxiQ3Jbk5STfnQZY0lswRSbMZ5uWYNwPPqipPDiVpscwRSTMM8+6YqwwOSfvIHJE0wzB7Qs5N8n7gH4DtexZW1emddSVp3JgjkmYYZgjZBNwGPHXKsgIMD0nDMkckzTDMu2NetBSNSBpf5oik2cw5hCR5VVW9OcnbGTxjuZOqenmnnUkaeeaIpPnMtydkz0Fk5y5FI/MpoIY5hHaBbr/bRPtFO6qbydWt1wSY7OZPwI5NHdW9ZV0ndc+94V6t1zxk1Y2t1wR4zNrLWq85Yzpoz7LJkZW3Fof8y/a933GBtj16Tes1AdZ08DmyOzal/aLAil2dlGX9Vd08Mq/ftqGTupfuf7fWa35rv0Nbrwmwdc03Wq85uYgkmXMIqaqPNhfPqapLFtuUpLsuc0TSfIY5MPWUJIcC/8LgbJfnVFX7I5SkcWaOSJphmANTH5dkNfAI4AnAx5LsV1Xt73eSNJbMEUmz2esQkuQxwGObr83AmcA5HfclaYyYI5JmM8zLMWczOKjsjcDHq2pHty1JGkPmiKQZhhlCDmRw+u3HAS9PMgl8qar+Z6edSRon5oikGYY5JuSGJJcAhwOHAY8GVnXdmKTxYY5Ims0wx4T8O/Bt4AvAO4EXuStV0kKYI5JmM8zLMfevqsnOO5E0zswRSTMM8zmk90zy4SRXJ7kqyYeSHNZ5Z5LGiTkiaYZhhpB3AWcA9wQOBT7aLJOkYZkjkmYYZgg5qKreVVW7mq9TgIM67kvSeDFHJM0wzBBybZITkkw0XycA13XdmKSxYo5ImmGYIeTFwC8BVwLbgF9olknSsMwRSTMM8zkhPwCevQS9SBpT5oik2QzzOSEHAS8Ftky9f1X5LEbSUMwRSbMZ5nNCPsLgRFOfAnZ3246kMWWOSJphmCFkfVX9XuedSBpn5oikGYY5MPXMJM9oc6VJNif5YJJ/TXJxkmPbrC9p2TFHJM0w556QJDcDBQT4/STbgZ3N9aqqTfuw3rcB/1RVv5BkNbB+H2pJWqbMEUnzmXMIqaqNXawwySYGp/N+YbOeHYAnspLGkDkiaT7DvDvmOOD8qrq1+YChY4C/bN5ytxj3Ba4B3pXkYcDXgFdU1a3T1nsicCLA6g0HMDmRRa5ubjXMi1HLRLX/6w/qTnRTd9Wte7/PYtx+azdnf//R9vafRN88ubb1mgB31K7WaxbVes2plkOOrNp4ANc9ZM2if4c5dbRtpoPT/a29vpt/567yaeembgqvvLWb8L99Zzf51IU7qv3HwuQiag7zL/EO4LZmQ38V8H3g3Qte04+tZBBA76iqhwO3Aq+efqeqOqmqtlbV1pVrN+zD6iQtA73nyMQ6c0RaboYZQnZVVQHPAd5WVW8D9mUX6w+BH1bVV5rrH2QQJpLGlzkiaYZhhpCbk7wGOAH4WJIJYNH7nKrqSuCyJEc1i54EfGux9SSNBHNE0gzDDCG/DGwHXtJs+IcCb9nH9f4GcFqSC4GjgT/Zx3qSljdzRNIMw5w75krgrVOu/wD4u31ZaVWdD2zdlxqSRoc5Imk2I/T+EEmSNE4cQiRJUi/mHUKSTCR5z1I1I2n8mCOS5jLvEFJVu4GDmo9ElqQFM0ckzWWYs+heCnwxyRkMPhAIgKp665w/IUl3dinmiKRphhlCrmi+VrBvHy4k6a7LHJE0wzBv0X09QJIN08/LIEnDMEckzWav745JcmySbwEXN9cfluR/d96ZpLFhjkiazTBv0f1L4KeB6wCq6gIGp9CWpGGZI5JmGOpzQqrqsmmLdnfQi6QxZo5Imm6YA1MvS/JooJq32L2cZpeqJA3JHJE0wzB7Qn4NeBmDE079kMGJol7WZVOSxo45ImmGYd4dcy3w/CXoRdKYMkckzWbOISTJ24Ga6/aqenknHUkaG+aIpPnMtyfk3CXrQtK4MkckzWnOIaSqTl3KRiSNH3NE0nz2ekxIkoOA3wMeBKzds7yqju+wrzsrSM25R3fRVt/SekkAaqg3Pi/Mil3t1wSgg78rwG2rJjqpy+50UnbH7vb7nezigQBMdvBv1s2j4MeWRY6sgF3r2i97+Ce7+QDYbcdtaL1mOsqRXeu72S67y71uyu6ebH+bv/SOA1uvCTCxqf2ai3kUDPMXO43BW+nuA7yewYmo/mUR65J012WOSJphmCHkwKo6GdhZVWdX1YuBR3Xcl6TxYo5ImmGYDyvb2XzfluRnGJwJ87DuWpI0hswRSTMMM4S8Icn+wG8Dbwc2Aa/stCtJ48YckTTDMB9WdmZz8Ubgid22I2kcmSOSZrPXY0KSHJnk00kuaq4/NMlru29N0rgwRyTNZpgDU/8P8Bqa13Sr6kLguV02JWnsmCOSZhhmCFlfVV+dtqyrd29LGk/miKQZhhlCrk1yBM3HuyT5BWBbp11JGjfmiKQZhnl3zMuAk4AHJLkc+B5wQqddSRo35oikGYZ5d8wlwJOTbABWVNXN3bclaZyYI5JmM+cQkuS35lgOQFW9taOeJI0Jc0TSfObbE7Kx+X4U8AjgjOb6s4DPd9mUpLFhjkia05xDSFW9HiDJWcAxe3afJvkj4ANL0p2kkWaOSJrPMO+OuRewY8r1HcCWTrqRNK7MEUkzDPPumHcDX03yYQZvr/s54NROu5I0bswRSTMM8+6Y/5XkH4HHNoteVFVf77YtSePEHJE0m2H2hFBV5wHntbniJBPAucDlVfXMNmtLWn7MEUnTDXNMSFdeAVzc4/oljT5zRBphvQwhSQ4Dfgb4v32sX9LoM0ek0dfXnpC/BF4FTPa0fkmjzxyRRtxQx4S0Kckzgaur6mtJnjDP/U4ETgRYvX4zE9triTrcdxM79n6fhaq0XxNgxe5u/q6Z7Kjujm7+EDt2tb8p3DG5qvWaALdW+//nTtbobF+wuBxZtd8BrL6p/V6uPHZD+0WB1Te1/2+yY2M328+qW7p5/NTKjvq9qZu61127ce93WqBNh93Rek2A3R3UXMyjoI89IccBz05yKfA+4Pgk75l+p6o6qaq2VtXWVWv2W+oeJS1vC86Rleu6GRYkLd6SDyFV9ZqqOqyqtgDPBT5TVZ5NU9LQzBFpPPT57hhJknQXtuTHhExVVZ8DPtdnD5JGmzkijS73hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF6s7LuBoRSs2NV+2UxW+0WBFTs6qJv2SwLURDeFJ+7opCwrdnb0h+jA+ontndTduGKi9ZoTGZ2/62LVCti9ppu6Xdi9uv1/k/0un2y9JsDO9R3lyG3dZPT2A7rpd93G9oPvpl1rW68J0H6KLO6/KfeESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXjiESJKkXiz5EJLk8CSfTXJxkm8mecVS9yBptJkj0nhY2cM6dwG/XVXnJdkIfC3JJ6vqWz30Imk0mSPSGFjyPSFVta2qzmsu3wxcDBy61H1IGl3miDQe+tgT8h+SbAEeDnxllttOBE4EWLtiPzb+8/eWtLd9cvsdrZfcffPNrdcEoKqTsms7qQr7P+GYTupe/90DW6/59qOe1npNgBufuL71mlftOrv1mktl6ByZ2Mjh77+09fXXrl2t1wQ6yREmJtqvCUzedlsndWv79k7qHnj39rd3gFsec0TrNT97/0e0XhNgzfPaf9xetetTC/6Z3g5MTbIf8CHgN6vqpum3V9VJVbW1qrauXrFu6RuUtOyZI9Jo62UISbKKQXCcVlWn99GDpNFmjkijr493xwQ4Gbi4qt661OuXNPrMEWk89LEn5DjgvwLHJzm/+XpGD31IGl3miDQGlvzA1Kr6ApClXq+k8WGOSOPBT0yVJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9cAiRJEm9SFX13cNeJbkZ+HbffQzp7sC1fTexAKPU7yj1CqPV71FVtbHvJrpkjnRqlPodpV5htPpdcI6s7KqTln27qrb23cQwkpw7Kr3CaPU7Sr3CaPWb5Ny+e1gC5khHRqnfUeoVRqvfxeSIL8dIkqReOIRIkqRejMoQclLfDSzAKPUKo9XvKPUKo9XvKPW6WKP0O45SrzBa/Y5SrzBa/S6415E4MFWSJI2fUdkTIkmSxoxDiCRJ6sWyG0KS/G2Sq5NcNGXZW5L8a5ILk3w4yeY+e9xjtl6n3PY7SSrJ3fvobTZz9ZvkN5J8O8k3k7y5r/6mmuNxcHSSLyc5P8m5SR7ZZ497JDk8yWeTXNz8DV/RLL9bkk8m+U7z/YC+e4V5+12W29limCPdGKUMAXOkS63lSFUtqy/gccAxwEVTlj0VWNlcfhPwpr77nKvXZvnhwCeA7wN377vPvfxtnwh8CljTXD+47z7n6fUs4OnN5WcAn+u7z6aXewDHNJc3Av8GPAh4M/DqZvmrl9Hjdq5+l+V21uLjZ1n+fqOUI6OUIfP0a4502++CtrNltyekqj4PXD9t2VlVtau5+mXgsCVvbBaz9dr4C+BVwLI66neOfn8d+NOq2t7c5+olb2wWc/RawKbm8v7AFUva1ByqaltVnddcvhm4GDgUeA5wanO3U4Gf7afDO5ur3+W6nS2GOdKNUcoQMEe61FaOLLshZAgvBv6x7ybmkuTZwOVVdUHfvQzpSOCxSb6S5Owkj+i7oXn8JvCWJJcBfwa8pud+ZkiyBXg48BXgkKraBoMNFji4v85mN63fqZb1dtaCZf37jViOjFKGgDnSun3JkZEaQpL8AbALOK3vXmaTZD3wB8Dr+u5lAVYCBwCPAn4X+Psk6belOf068MqqOhx4JXByz/3cSZL9gA8Bv1lVN/Xdz97M1e9y38721XL//UYwR0YpQ8AcadW+5sjIDCFJXgA8E3h+NS82LUNHAPcBLkhyKYPdUOcl+Yleu5rfD4HTa+CrwCSDEyYtRy8ATm8ufwBYFgeUASRZxWBDPK2q9vR4VZJ7NLffA1g2u6nn6HdUtrNFG5Hfb9RyZJQyBMyR1rSRIyMxhCR5GvB7wLOr6ra++5lLVX2jqg6uqi1VtYXBxnlMVV3Zc2vz+QfgeIAkRwKrWb5nbLwCeHxz+XjgOz328h+aZ30nAxdX1Vun3HQGg8Cj+f6Rpe5tNnP1Oyrb2WKNyu83gjkyShkC5kgrWsuRvo+wnf4FvBfYBuxksPG9BPgucBlwfvP1zr77nKvXabdfyjI5qn2ev+1q4D3ARcB5wPF99zlPr48BvgZcwOC1x5/su8+m18cwONjtwimP0WcABwKfZhBynwbu1neve+l3WW5nLT5+luXvN0o5MkoZMk+/5ki3/S5oO/Nj2yVJUi9G4uUYSZI0fhxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCRJLPJdnadx+SRpc5osVwCFHnkkz03YOk0WaOjCeHkLuIJFuS/GuSU5NcmOSDzTkqpt/vHUnOTfLNJK9vlj0pyYen3OcpSU5vLj81yZeSnJfkA815BEhyaZLXJfkC8IvT1vGLSS5KckGSzzfLXpjkr6fc58wkT2gu35LkTUm+luRTSR7ZPOu6pDnRl6QlYI6obQ4hdy1HASdV1UOBm4D/Pst9/qCqtgIPBR6f5KHAZ4AHJjmouc+LgHcluTvwWuDJVXUMcC7wW1Nq3VFVj6mq901bx+uAn66qhwHDbPwbgM9V1U8CNwNvAJ4C/Bzwx0P8vKT2mCNqjUPIXctlVfXF5vJ7GHzs7nS/lOQ84OvAg4EH1eBjdd8NnJBkM3Asg9MzPwp4EPDFJOczOK/BvafUev8cfXwROCXJS4FhdrHuAP6pufwN4Oyq2tlc3jLEz0tqjzmi1qzsuwEtqemf0X+n60nuA/wO8Iiq+lGSU4C1zc3vAj4K3AF8oKp2NScw+mRVPW+O9d06axNVv5bkp4CfAc5PcjSDUz5PHYrXTrm8s358foFJYHtTZzKJj2FpaZkjao17Qu5a7pXk2Oby84AvTLt9E4MN/sYkhwBP33NDVV3B4OyTrwVOaRZ/GTguyf0AkqxvzqI5ryRHVNVXqup1DM62eTiDk3QdnWRFksNZRqfXlnQn5oha4/R313Ix8IIkf8PgjIzvmHpjVV2Q5OvAN4FLGOzunOo04KCq+lZz/2uSvBB4b5I1zX1eC/zbXvp4S5L7A2FwVsgLmuXfY7BrdM/ZOCUtP+aIWuNZdO8ikmwBzqyqh+xDjb8Gvl5VJ7fVl6TRYY6obe4J0VCSfI3BLtbf7rsXSaPJHNF07gmRJEm98MBUSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUi/8P+5Fl3j/dNpoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 离线回合更新\n",
    "重要性采样策略评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_monte_carlo_importance_resample(env, policy, behavior_policy,\n",
    "        episode_num=500000):\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # 用行为策略玩一回合\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n,\n",
    "                    p=behavior_policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # 玩好了\n",
    "        g = reward # 回报\n",
    "        rho = 1. # 重要性采样比率\n",
    "        for state, action in reversed(state_actions):\n",
    "            c[state][action] += rho\n",
    "            q[state][action] += (rho / c[state][action] * (g - q[state][action]))\n",
    "            rho *= (policy[state][action] / behavior_policy[state][action])\n",
    "            if rho == 0:\n",
    "                break # 提前终止\n",
    "    return q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy = np.zeros((22, 11, 2, 2))\n",
    "policy[20:, :, :, 0] = 1 # >= 20 时收手\n",
    "policy[:20, :, :, 1] = 1 # < 20 时继续\n",
    "behavior_policy = np.ones_like(policy) * 0.5\n",
    "q = evaluate_monte_carlo_importance_resample(env, policy, behavior_policy)\n",
    "v = (q * policy).sum(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHsdJREFUeJzt3XmYZAV57/Hvb3r2YYZhExHRMQpGzYNIRiNxxyVqXJLcLObKvW6RLD7RmMVoYsw1jzdGTUyMea6Ge4kQ5WqiokHN4hZxuYpBBESJ0SAKMrI5wAjjTM/0e/+oM0nT21T3nNOnq/h+nqefqTpV9Z63ij4/3j51qk6qCkmSpOW2qu8GJEnSXZNDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDiCRJ6oVDyF1Qku8l+YEFbr86yROWsydJ48m80UIcQu6CquqwqroKIMk5SV6zHOtN8twkn16OdUlaGfrKG40GhxBJktQLh5AxkeR5ST4w7frXk/zttOvXJDmluVxJ7pfkTODZwMuaXaYfmFbylCSXJ7k1yd8kWT+t1gub+t9NckGSezTLtzW1V0+77yeS/EKSBwBvBU5r1nXLAs/jyiS7klyV5Bdn3P7MJJcmuS3Jvyd5crP88CRnJ9mR5NtJXpNk4hBeUknzWM68mbHe+yb5eJKbk9yU5LwkW6fdfkKS85Pc2NznL6bd9vwmW3Ym+ack927xJdESOYSMjwuBRyVZleQ4YA3wCIDm/djDgMunP6CqzgLOA17f7DJ9+rSbfxZ4MnAf4GTguU2t04HXNrcfB3wTeNfBmquqK4FfAj7brGvrPHe9AXgasAV4HvCnSU5t1v0w4K+B3wK2Ao8Grm4edy6wD7gf8BDgScAvHKwvSUuyLHkzhzDIn3sADwBOAP5Hs94J4IMMMmkbcDxNNiX5CeB3gJ8CjgE+Bbxzic9dLVp98LtoFFTVVUl2AacAJwH/xOCvix8ETgM+VVVTiyj551V1HUDzF8spzfJnA39VVZc0t70C2JlkW0vP40PTrl6Y5MPAo4BLgBc06/5Ic/u3mx6OBZ4CbK2q3cDtSf4UOBP4yzb6kvSfljFvZq7368DXm6s3Jnkj8PvN9YcxGE5+q6r2NcsOHIP2i8Brmz+GSPKHwO8kuXdVfXMRfaplDiHj5ULgsQz2BlwI3AI8hkEoXLjIWt+ZdvkOBhs3zb+XHLihqr6X5GYGf3V8e0ldT5PkKQxC5SQGe+o2Al9qbj4B+Ps5HnZvBn+J7UhyYNkq4JpD7UfSvJYjb+4kyd2AP2fwh8lmBtv5zubmE4BvThtAprs38KYkfzK9HIPccgjpkW/HjJcDofCo5vKFDELhMcwfCos9jfJ1DDZoAJJsAo5iMIDc3izeOO3+dx92XUnWAe8F/hg4tnnL5u8ZhAUMhor7zvHQa4A9wNFVtbX52VJVDxr6WUlarOXIm5le29Q4uaq2AGdw53y41/Rj0qa5BvjFafmwtao2VNX/O8R+dIgcQsbLhcDjgA1VdS2D9z2fzGBI+OI8j7kemPcz/HP4v8DzkpzSDA1/CFxUVVdX1Y0MhpEzkkwkeT53HhquB+6ZZO08tdcC64AbgX3NXpEnTbv97Gbdj2/eiz4+yQ9W1Q7gw8CfJNnS3HbfJI9ZxPOStDjLkTczbQa+B9yS5HgGx4cd8HlgB/BHSTYlWZ/kEc1tbwVekeRB8B8Hsv/MIfShljiEjJGq+jcGG+inmuu3AVcBn6mq/fM87GzggUluSfL+IdbxMeD3GOyx2MFgyHjWtLu8kEEw3Aw8CJj+l8bHgS8D30ly0xy1dwEvBv6WwS7W/wpcMO32z9McrArcyiAED+yV+e8MhpivNI99D4MDZyV1YDnyZg6vBk5lsP1/CDh/Wj/7gaczeHvoW8C1wM81t70PeB3wriS3AVcwOI5MPUvVoe4dkyRJWjz3hEiSpF44hEiSpF44hEiSpF44hEiSpF6MxJeVrV29sTasObz9wl0dk7uoLwoctmb7JQd1Oys8WmWnOvhvdmI3m9cPrN958Dst0jXX7OPm707l4PccXWvXbKr16+c7W8DS1apuXrbsb/+XvSY66nWqmw0z+zrYLrvUwcuw/17dvLb32/Dd1msuJUdGYgjZsOZwTrvfC9ovPDnXF+sduuzZ237RffN94u3Q1L5uXgP2d9MvHfU7tfv77Rf9y6Pbrwm888R3t17z8U+9sfWaK8369Vt52Cm/0nrdycO6idE1t022XnPy8Pm+oufQrP5e+70CrNm5u5O6nf3x1UFO3/bn3fT6/ge9vfWaT3rqrG9eOCjfjpEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb3obAhJ8ldJbkhyxbRlRyb5SJKvNf8e0dX6JY0+c0Qab13uCTkHePKMZS8HPlZVJwIfa65L0nzOwRyRxlZnQ0hVfRL47ozFzwTObS6fC/xEV+uXNPrMEWm8rV7m9R1bVTsAqmpHkrvNd8ckZwJnAqxbdziTR25svZnVt+5uvSZA9k62XrMOP6z1mgB00CtApqqTunX7HZ3UXdVBvy+79z+0XhPgiIkOtoXROjxsiTmylam17T/PVfu6+V2/+eT2/ztvvHGq9ZoAk5vWdVJ3z1FrO6m7/sY9ndTNvvZf39eceF7rNQGOWLWh9ZoTZNGPWbHJU1VnVdX2qtq+ds2mvtuRNILulCNrzRFppVnuIeT6JMcBNP/esMzrlzT6zBFpTCz3EHIB8Jzm8nOAv1vm9UsafeaINCa6/IjuO4HPAvdPcm2SFwB/BDwxydeAJzbXJWlO5og03jo7MLWqfn6emx7f1ToljRdzRBpvK/bAVEmSNN4cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi9W993A0KraL7m2m6c/ddjG1mtm//7WawLk+3s7qUvSTd09e7qpu2F96yU3r/p+6zUB9lf7v7dF+9vXSrNvXdh54rrW6+49vJvf9S1Xt7/Nr7+xm+19cnM3WbrzxDWd1N18+fWd1N118rGt17xh/+bWaw7s7Kju4rgnRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9aKXISTJS5N8OckVSd6ZZH0ffUgaXeaINPqWfQhJcjzwYmB7Vf0QMAE8a7n7kDS6zBFpPPT1dsxqYEOS1cBG4Lqe+pA0uswRacStXu4VVtW3k/wx8C1gN/Dhqvrwgg+agok9+1vvZdXuydZrdiW77uim8No13dTdOzqvLUDdsbv1mt+vbjavibT/t0NI6zW7tJQcWbNrkrt/dEfrvXzj2fdovSbAqn3t19x9t7XtFwXW7mo/nwGO++StndSdvPvWTuquvr391+Goie+1XhNWTo708XbMEcAzgfsA9wA2JTljjvudmeTiJBdP7rt9uduUtIItJUf27u9okJe0ZH28HfME4BtVdWNVTQLnAz86805VdVZVba+q7WtWb1r2JiWtaIvOkbUTG5e9SUkL62MI+Rbw8CQbkwR4PHBlD31IGl3miDQGln0IqaqLgPcAlwBfano4a7n7kDS6zBFpPCz7gakAVfX7wO/3sW5J48EckUaf35gqSZJ64RAiSZJ6cdC3Y5L81ByLbwW+VFU3tN+SpHFjjkiayzDHhLwAOA345+b6Y4HPAScl+YOqentHvUkaH+aIpFmGGUKmgAdU1fUASY4F3gL8CPBJwPCQdDDmiKRZhjkmZNuB4GjcAJxUVd8FRuu7uSX1xRyRNMswe0I+leSDwLub6/8F+GSSTcAtnXUmaZyYI5JmGWYIeRGDwHgEEOCvgfdWVQGP67A3SePDHJE0y0GHkCYk3tP8SNKimSOS5nLQY0KS/FSSryW5NcltSXYluW05mpM0HswRSXMZ5u2Y1wNPrypPDiVpqcwRSbMM8+mY6w0OSYfIHJE0yzB7Qi5O8jfA+4E9BxZW1fmddSVp3JgjkmYZZgjZAtwBPGnasgIMD0nDMkckzTLMp2OetxyNSBpf5oikucw7hCR5WVW9PsmbGfzFcidV9eJOO5M08swRSQtZaE/IgYPILl6ORhYUmFozzDG0i7RhTfs1gVW37zn4nRapNm9svSZAdrffKwA16/837Vi3rpOy2dP+63DN5FGt1wTYs679k85OzZ4P2rJicmT/xjXc8sPHtl737hftbb0mwB3HDPNu+eKsv2V/6zUB9m3sIJ+BDTt3dVKXo7d0UnbNRy9rveZ39m1tvSbADfu/1XrNSaYW/Zh5f8ur6gPNxU9V1VVLbUrSXZc5Imkhw4za5yQ5HvgXBme7/FRVfanbtiSNGXNE0izDHJj66CRrgYcCjwU+lOSwqjqy6+YkjQdzRNJcDjqEJHkk8KjmZyvwQeBTHfclaYyYI5LmMszbMRcyOKjstcDfV1U3R2FJGmfmiKRZhhlCjmJw+u1HAy9OMgV8tqp+r9POJI0Tc0TSLMMcE3JLkquAE4B7Aj8KdPPZVkljyRyRNJdhjgn5d+CrwKeBtwLPc1eqpMUwRyTNZZi3Y06sqsV/A4kk/SdzRNIsw3zN3T2SvC/JDUmuT/LeJPfsvDNJ48QckTTLMEPI24ALgHsAxwMfaJZJ0rDMEUmzDDOEHFNVb6uqfc3POcAxHfclabyYI5JmGWYIuSnJGUkmmp8zgJu7bkzSWDFHJM0yzBDyfOBnge8AO4CfbpZJ0rDMEUmzDPM9Id8CnrEMvUgaU+aIpLkM8z0hxwAvBLZNv39V+VeMpKGYI5LmMsz3hPwdgxNNfRTY3207ksaUOSJplmGGkI1V9duddyJpnJkjkmYZ5sDUDyZ5apsrTbI1yXuS/GuSK5Oc1mZ9SSuOOSJplnn3hCTZBRQQ4HeS7AEmm+tVVVsOYb1vAv6xqn46yVpg4yHUkrRCmSOSFjLvEFJVm7tYYZItDE7n/dxmPXsBT2QljSFzRNJChvl0zCOAS6vq9uYLhk4F/qz5yN1S/ABwI/C2JA8GvgC8pKpun7HeM4EzAdatO5xalSWubn6rbt/Tek2AWjvMoTaLs+qGna3XBKhNGzqpmz3d/P8gq4Z5B3HxqoOae2uig6qwv7rotlsrIUfWr97C4V+8YcnPYT63PfhurdcE2HLV7tZrTuyebL0mwN6jusmR2x5yXCd1t1x2fSd173jyQ1uvuSbfaL0mwLq0n6VLqTjMY94C3NFs6C8Dvgm8fQnrOmA1gwB6S1U9BLgdePnMO1XVWVW1vaq2r12z6RBWJ2kF6D9HJrr5H6WkpRtmCNlXVQU8E3hTVb0JOJRdrNcC11bVRc319zAIE0njyxyRNMswQ8iuJK8AzgA+lGQCWLPUFVbVd4Brkty/WfR44CtLrSdpJJgjkmYZZgj5OWAP8IJmwz8eeMMhrvdXgfOSXA6cAvzhIdaTtLKZI5JmGebcMd8B3jjt+reAvz6UlVbVpcD2Q6khaXSYI5Lm0s1HDSRJkg7CIUSSJPViwSEkyUSSdyxXM5LGjzkiaT4LDiFVtR84pvlKZElaNHNE0nyG+WrPq4HPJLmAwRcCAVBVb5z3EZJ0Z1djjkiaYZgh5LrmZxWH9uVCku66zBFJswzzEd1XAyTZNPO8DJI0DHNE0lwO+umYJKcl+QpwZXP9wUn+V+edSRob5oikuQzzEd0/A34MuBmgqi5jcAptSRqWOSJplqG+J6SqrpmxaH8HvUgaY+aIpJmGOTD1miQ/ClTzEbsX0+xSlaQhmSOSZhlmT8gvAS9icMKpaxmcKOpFXTYlaeyYI5JmGebTMTcBz16GXiSNKXNE0lzmHUKSvBmo+W6vqhd30pGksWGOSFrIQntCLl62LiSNK3NE0rzmHUKq6tzlbETS+DFHJC3koMeEJDkG+G3ggcD6A8ur6vQO+7qzKZjY0/6n+abWd3M+rVW77mi/6MRE+zWB7N7TSV1q3j3whybppu5U+/3umtrQek2APbWv9ZpT879j0oqVkCO1eoL9R7f/jfGZar0kADXRwe/6VDfNrrm1mxxZta+bfvcfsamTuhuvuK71mtfuPar1mgB7NrTf61L+aw3z6ZjzGHyU7j7AqxmciOpflrAuSXdd5oikWYYZQo6qqrOByaq6sKqeDzy8474kjRdzRNIsw3xZ2WTz744kP87gTJj37K4lSWPIHJE0yzBDyGuSHA78BvBmYAvw0k67kjRuzBFJswzzZWUfbC7eCjyu23YkjSNzRNJcDnpMSJKTknwsyRXN9ZOTvLL71iSNC3NE0lyGOTD1fwOvoHlPt6ouB57VZVOSxo45ImmWYYaQjVX1+RnL2v+iAknjzByRNMswQ8hNSe5Lc/6HJD8N7Oi0K0njxhyRNMswn455EXAW8INJvg18Azij064kjRtzRNIsw3w65irgCUk2Aauqalf3bUkaJ+aIpLnMO4Qk+fV5lgNQVW/sqCdJY8IckbSQhfaEHDjT0/2BhwIXNNefDnyyy6YkjQ1zRNK85h1CqurVAEk+DJx6YPdpkv8BvHtZupM00swRSQsZ5tMx9wL2Tru+F9jWSTeSxpU5ImmWYT4d83bg80nex+DjdT8JnNtpV5LGjTkiaZZhPh3zP5P8A/CoZtHzquqL3bYlaZyYI5LmMsyeEKrqEuCSNlecZAK4GPh2VT2tzdqSVh5zRNJMwxwT0pWXAFf2uH5Jo88ckUZYL0NIknsCPw78nz7WL2n0mSPS6OtrT8ifAS8Dpnpav6TRZ45II26oY0LalORpwA1V9YUkj13gfmcCZwKsX72FVTu/134zExPt1wRYu6b9mvv2t1+zy7odvbZTO2/ppG42bmi95lR1M+NPDs4B16r2K3ZrKTmydtMR3HLiptZ72XT9ZOs1AdZcf1vrNe846ajWawKsu3lPJ3Wzt5v5cmLn7Z3UZc2y/y91ydaQ1msupWIfe0IeATwjydXAu4DTk7xj5p2q6qyq2l5V29dOtP8/CEkjbdE5snp9+wOIpEOz7ENIVb2iqu5ZVduAZwEfryrPpilpaOaINB76/HSMJEm6C+v1Dayq+gTwiT57kDTazBFpdLknRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9cIhRJIk9WJ13w0MoyZWMbV5Y+t1V+2ZbL0mAFXd1O3CREdz6FQ3r8GqI4/opO7Ud3e2XnOyJlqvCbCGtF6z/Yorz9QEfP/I9p/p4Vfta70mQHbvab3mxqtuab0mADfc3EnZbD6sk7pTW7upm8n9rde8Y2pt6zUBJmk/o5dS0T0hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpF8s+hCQ5Ick/J7kyyZeTvGS5e5A02swRaTys7mGd+4DfqKpLkmwGvpDkI1X1lR56kTSazBFpDCz7npCq2lFVlzSXdwFXAscvdx+SRpc5Io2HPvaE/Ick24CHABfNcduZwJkA6yc2M3HTra2vv/bubb0mQNasab/o/v3t1wRq9+5O6jJV3ZTds6eTutVB3Td/7vTWawLsf3har3nT/s+2XnO5DJsjq7ccwb6Ny9raIdl3jyNbrzm1dqL1mgCr13WQeQCT3eTe7dsO66Tu5ou+1XrNsz/7qNZrAkw+vP3fhRv3fWLRj+ntwNQkhwHvBX6tqm6beXtVnVVV26tq+9qJDcvfoKQVbzE5snrjpuVvUNKCehlCkqxhEBznVdX5ffQgabSZI9Lo6+PTMQHOBq6sqjcu9/oljT5zRBoPfewJeQTw34DTk1za/Dy1hz4kjS5zRBoDy35galV9Gmj/yDpJdxnmiDQe/MZUSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUC4cQSZLUi1RV3z0cVJJdwFf77mNIRwM39d3EIoxSv6PUK4xWv/evqs19N9Elc6RTo9TvKPUKo9XvonNkdVedtOyrVbW97yaGkeTiUekVRqvfUeoVRqvfJBf33cMyMEc6Mkr9jlKvMFr9LiVHfDtGkiT1wiFEkiT1YlSGkLP6bmARRqlXGK1+R6lXGK1+R6nXpRql5zhKvcJo9TtKvcJo9bvoXkfiwFRJkjR+RmVPiCRJGjMOIZIkqRcrbghJ8ldJbkhyxbRlb0jyr0kuT/K+JFv77PGAuXqddttvJqkkR/fR21zm6zfJryb5apIvJ3l9X/1NN8/vwSlJPpfk0iQXJ3lYnz0ekOSEJP+c5MrmNXxJs/zIJB9J8rXm3yP67hUW7HdFbmdLYY50Y5QyBMyRLrWWI1W1on6ARwOnAldMW/YkYHVz+XXA6/ruc75em+UnAP8EfBM4uu8+D/LaPg74KLCuuX63vvtcoNcPA09pLj8V+ETffTa9HAec2lzeDPwb8EDg9cDLm+UvX0G/t/P1uyK3sxZ/f1bk8xulHBmlDFmgX3Ok234XtZ2tuD0hVfVJ4Lszln24qvY1Vz8H3HPZG5vDXL02/hR4GbCijvqdp99fBv6oqvY097lh2Rubwzy9FrCluXw4cN2yNjWPqtpRVZc0l3cBVwLHA88Ezm3udi7wE/10eGfz9btSt7OlMEe6MUoZAuZIl9rKkRU3hAzh+cA/9N3EfJI8A/h2VV3Wdy9DOgl4VJKLklyY5KF9N7SAXwPekOQa4I+BV/TczyxJtgEPAS4Cjq2qHTDYYIG79dfZ3Gb0O92K3s5asKKf34jlyChlCJgjrTuUHBmpISTJ7wL7gPP67mUuSTYCvwu8qu9eFmE1cATwcOC3gL9Nkn5bmtcvAy+tqhOAlwJn99zPnSQ5DHgv8GtVdVvf/RzMfP2u9O3sUK305zeCOTJKGQLmSKsONUdGZghJ8hzgacCzq3mzaQW6L3Af4LIkVzPYDXVJkrv32tXCrgXOr4HPA1MMTpi0Ej0HOL+5/G5gRRxQBpBkDYMN8byqOtDj9UmOa24/Dlgxu6nn6XdUtrMlG5HnN2o5MkoZAuZIa9rIkZEYQpI8Gfht4BlVdUff/cynqr5UVXerqm1VtY3BxnlqVX2n59YW8n7gdIAkJwFrWblnbLwOeExz+XTgaz328h+av/rOBq6sqjdOu+kCBoFH8+/fLXdvc5mv31HZzpZqVJ7fCObIKGUImCOtaC1H+j7CduYP8E5gBzDJYON7AfB14Brg0ubnrX33OV+vM26/mhVyVPsCr+1a4B3AFcAlwOl997lAr48EvgBcxuC9xx/uu8+m10cyONjt8mm/o08FjgI+xiDkPgYc2XevB+l3RW5nLf7+rMjnN0o5MkoZskC/5ki3/S5qO/Nr2yVJUi9G4u0YSZI0fhxCJElSLxxCJElSLxxCJElSLxxCJElSLxxCRJJPJNnedx+SRpc5oqVwCFHnkkz03YOk0WaOjCeHkLuIJNuS/GuSc5NcnuQ9zTkqZt7vLUkuTvLlJK9ulj0+yfum3eeJSc5vLj8pyWeTXJLk3c15BEhydZJXJfk08DMz1vEzSa5IclmSTzbLnpvkL6bd54NJHttc/l6S1yX5QpKPJnlY81fXVc2JviQtA3NEbXMIuWu5P3BWVZ0M3Ab8yhz3+d2q2g6cDDwmycnAx4EHJDmmuc/zgLclORp4JfCEqjoVuBj49Wm1vl9Vj6yqd81Yx6uAH6uqBwPDbPybgE9U1Q8Du4DXAE8EfhL4gyEeL6k95oha4xBy13JNVX2mufwOBl+7O9PPJrkE+CLwIOCBNfha3bcDZyTZCpzG4PTMDwceCHwmyaUMzmtw72m1/maePj4DnJPkhcAwu1j3Av/YXP4ScGFVTTaXtw3xeEntMUfUmtV9N6BlNfM7+u90Pcl9gN8EHlpVO5OcA6xvbn4b8AHg+8C7q2pfcwKjj1TVz8+zvtvnbKLql5L8CPDjwKVJTmFwyufpQ/H6aZcn6z/PLzAF7GnqTCXxd1haXuaIWuOekLuWeyU5rbn888CnZ9y+hcEGf2uSY4GnHLihqq5jcPbJVwLnNIs/Bzwiyf0AkmxszqK5oCT3raqLqupVDM62eQKDk3SdkmRVkhNYQafXlnQn5oha4/R313Il8Jwkf8ngjIxvmX5jVV2W5IvAl4GrGOzunO484Jiq+kpz/xuTPBd4Z5J1zX1eCfzbQfp4Q5ITgTA4K+RlzfJvMNg1euBsnJJWHnNErfEsuncRSbYBH6yqHzqEGn8BfLGqzm6rL0mjwxxR29wToqEk+QKDXay/0XcvkkaTOaKZ3BMiSZJ64YGpkiSpFw4hkiSpFw4hkiSpFw4hkiSpFw4hkiSpF/8f5q55okXBU2IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "重要性采样回合更新"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monte_carlo_importance_resample(env, episode_num=500000):\n",
    "    policy = np.zeros((22, 11, 2, 2))\n",
    "    policy[:, :, :, 0] = 1.\n",
    "    behavior_policy = np.ones_like(policy) * 0.5 # 柔性策略\n",
    "    q = np.zeros_like(policy)\n",
    "    c = np.zeros_like(policy)\n",
    "    for _ in range(episode_num):\n",
    "        # 用行为策略玩一回合\n",
    "        state_actions = []\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            state = ob2state(observation)\n",
    "            action = np.random.choice(env.action_space.n,\n",
    "                    p=behavior_policy[state])\n",
    "            state_actions.append((state, action))\n",
    "            observation, reward, done, _ = env.step(action)\n",
    "            if done:\n",
    "                break # 玩好了\n",
    "        g = reward # 回报\n",
    "        rho = 1. # 重要性采样比率\n",
    "        for state, action in reversed(state_actions):\n",
    "            c[state][action] += rho\n",
    "            q[state][action] += (rho / c[state][action] * (g - q[state][action]))\n",
    "            # 策略改进\n",
    "            a = q[state].argmax()\n",
    "            policy[state] = 0.\n",
    "            policy[state][a] = 1.\n",
    "            if a != action: # 提前终止\n",
    "                break\n",
    "            rho /= behavior_policy[state][action]\n",
    "    return policy, q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "policy, q = monte_carlo_importance_resample(env)\n",
    "v = q.max(axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGhJJREFUeJzt3Xm0ZHV57vHvQzeIjSAKLaGbxjYoTlnYktaIiAMOOGu8avTKvYpGNNcV1BgH1GjM8joHY3RdDQkKKnFCVEQTUJRBr0LaZrYVDaIMzagM4pVB3vtH7TaHM3WdPqfqd6r6+1nrrK7ae5/a7+nV9fRzdu2qnapCkiRp2LZpPYAkSdo6WUIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWkK1Qkl8n+cNZ1l+S5AnDnEnSeDJvNBtLyFaoqu5WVRcDJDk6yTuHsd8kL0nynWHsS9Li0CpvNBosIZIkqQlLyJhIckiSr064/9Mkn59w/9Ika7rbleS+SQ4FXgS8oTtk+tUJD7kmyXlJbkjyuSTbT3isl3eP/8skJyRZ0S1f3T320gnbnprkz5M8EPgYsF+3r+tn+Tk2JLkpycVJXjFp/bOSnJPkxiT/meTJ3fK7JzkqycYklyd5Z5Il8/grlTSDYebNpP3uleRbSa5Lcm2SY5PsPGH9qiTHJ7mm2+YjE9a9tMuWXyU5Kcm9F/CvRFvIEjI+TgMOSLJNkt2BbYH9AbrXY+8GnDfxG6rqSOBY4H3dIdNnTFj9fODJwH2AfYCXdI91IPDubv3uwM+Bz25uuKraALwS+F63r51n2PRq4OnATsAhwAeT7Nvt++HAJ4HXAzsDjwYu6b7vGOB24L7AQ4EnAX++ubkkbZGh5M00Qi9/VgAPBFYBf9vtdwlwIr1MWg2spMumJM8G3gw8B1gOnAF8Zgt/di2gpZvfRKOgqi5OchOwBtgbOInebxcPAPYDzqiqO+bwkP9YVVcAdL+xrOmWvwj4eFWt79YdDvwqyeoF+jm+NuHuaUlOBg4A1gMv6/b9jW795d0MuwFPAXauqv8H3Jzkg8ChwD8txFyS/ssQ82byfn8K/LS7e02SI4C3d/cfTq+cvL6qbu+WbToH7RXAu7tfhkjyLuDNSe5dVT+fw5xaYJaQ8XIa8Fh6RwNOA64HHkMvFE6b42NdOeH2b+g9uen+XL9pRVX9Osl19H7ruHyLpp4gyVPohcre9I7ULQPO71avAr4+zbfdm95vYhuTbFq2DXDpfOeRNKNh5M2dJLkX8I/0fjHZkd7z/Ffd6lXAzycUkInuDXwoyd9PfDh6uWUJaciXY8bLplA4oLt9Gr1QeAwzh8JcL6N8Bb0nNABJdgB2oVdAbu4WL5uw/R/0u68kdwG+CHwA2K17yebr9MICeqVir2m+9VLgFmDXqtq5+9qpqh7c908laa6GkTeTvbt7jH2qaifgYO6cD3tOPCdtgkuBV0zIh52r6q5V9X/nOY/myRIyXk4DHgfctaouo/e655PplYSzZ/ieq4AZ38M/jX8FDkmypisN7wLOrKpLquoaemXk4CRLkryUO5eGq4A9kmw3w2NvB9wFuAa4vTsq8qQJ64/q9v347rXolUkeUFUbgZOBv0+yU7durySPmcPPJWluhpE3k+0I/Bq4PslKeueHbXIWsBF4T5IdkmyfZP9u3ceAw5M8GH5/Ivvz5jGHFoglZIxU1UX0nqBndPdvBC4GvltVv5vh244CHpTk+iRf7mMfpwB/Q++IxUZ6JeMFEzZ5Ob1guA54MDDxN41vARcCVya5dprHvgk4DPg8vUOs/x04YcL6s+hOVgVuoBeCm47K/E96JeaH3fceR+/EWUkDMIy8mcY7gH3pPf+/Bhw/YZ7fAc+g9/LQL4DLgD/r1n0JeC/w2SQ3AhfQO49MjaVqvkfHJEmS5s4jIZIkqQlLiCRJasISIkmSmrCESJKkJkbiw8p2veeSWr1q29ZjaA4uOm/Z5jfSovFbbubWuiWb33J0mSMatK0997YkR0aihKxetS1nnbSq9Riag4NWTPupy1qkzqxTWo8wcOaIBm1rz70tyRFfjpEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTQyshCT5eJKrk1wwYdk9k3wjyU+6P+8xqP1LGn3miDTeBnkk5GjgyZOWvQk4paruB5zS3ZekmRyNOSKNrYGVkKo6HfjlpMXPAo7pbh8DPHtQ+5c0+swRabwtHfL+dquqjQBVtTHJvWbaMMmhwKEAe64c9piSFjFzRPNy0Io1rUdQZ9GemFpVR1bV2qpau3yXJa3HkTSCzBFpcRt2Cbkqye4A3Z9XD3n/kkafOSKNiWGXkBOAF3e3Xwx8Zcj7lzT6zBFpTAzyLbqfAb4H3D/JZUleBrwHeGKSnwBP7O5L0rTMEWm8DexMrap64QyrHj+ofUoaL+aINN4W7YmpkiRpvFlCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhNNSkiS1ya5MMkFST6TZPsWc0gaXeaINPqGXkKSrAQOA9ZW1R8BS4AXDHsOSaPLHJHGQ6uXY5YCd02yFFgGXNFoDkmjyxyRRtzQS0hVXQ58APgFsBG4oapOHvYckkaXOSKNhxYvx9wDeBZwH2AFsEOSg6fZ7tAk65Ksu+a63w17TEmLmDkijYcWL8c8AfhZVV1TVbcBxwOPnLxRVR1ZVWurau3yXZYMfUhJi5o5Io2BFiXkF8AjkixLEuDxwIYGc0gaXeaINAZanBNyJnAcsB44v5vhyGHPIWl0mSPSeFjaYqdV9Xbg7S32LWk8mCPS6PMTUyVJUhOWEEmS1MRmX45J8pxpFt8AnF9VVy/8SJLGjTkiaTr9nBPyMmA/4Nvd/ccC3wf2TvJ3VfWpAc0maXyYI5Km6KeE3AE8sKquAkiyG/BR4E+A0wHDQ9LmmCOSpujnnJDVm4KjczWwd1X9ErhtMGNJGjPmiKQp+jkSckaSE4EvdPf/G3B6kh2A6wc2maRxYo5ImqKfEvIqeoGxPxDgk8AXq6qAxw1wNknjwxyRNMVmS0gXEsd1X5I0Z+aIpOls9pyQJM9J8pMkNyS5MclNSW4cxnCSxoM5Imk6/bwc8z7gGVXlxaEkbSlzRNIU/bw75iqDQ9I8mSOSpujnSMi6JJ8DvgzcsmlhVR0/sKkkjRtzRNIU/ZSQnYDfAE+asKwAw0NSv8wRSVP08+6YQ4YxiKTxZY5Ims6MJSTJG6rqfUk+TO83ljupqsMGOpmkkWeOSJrNbEdCNp1Etm4Yg8zmovOWcdCKNa3HkDR3iyZHpE1OuuKcgTyu/0/N3YwlpKq+2t08o6ouHtI8ksaIOSJpNv2cmHp0kpXAf9C72uUZVXX+YMeSNGbMEUlT9HNi6qOTbAc8DHgs8LUkd6uqew56OEnjwRyRNJ3NlpAkjwIO6L52Bk4EzhjwXJLGiDkiaTr9vBxzGr2Tyt4NfL2qbh3sSJLGkDkiaYp+Ssgu9C6//WjgsCR3AN+rqr8Z6GSSxok5ImmKfs4JuT7JxcAqYA/gkcC2gx5M0vgwRyRNp59zQv4T+DHwHeBjwCEeSpU0F+aIpOn083LM/arqjoFPImmcmSOSptimj21WJPlSkquTXJXki0n2GPhkksaJOSJpin5KyCeAE4AVwErgq90ySeqXOSJpin5KyPKq+kRV3d59HQ0sH/BcksaLOSJpin5KyLVJDk6ypPs6GLhu0INJGivmiKQp+ikhLwWeD1wJbASe2y2TpH6ZI5Km6OdzQn4BPHMIs0gaU+aIpOn08zkhy4GXA6snbl9V/hYjqS/miKTp9PM5IV+hd6GpbwK/G+w4ksaUOSJpin5KyLKqeuPAJ5E0zswRSVP0c2LqiUmeupA7TbJzkuOS/CjJhiT7LeTjS1p0zBFJU8x4JCTJTUABAd6c5Bbgtu5+VdVO89jvh4B/r6rnJtkOWDaPx5K0SJkjkmYzYwmpqh0HscMkO9G7nPdLuv3cCnghK2kMmSOSZrPZl2OS7J9kh+72wUmOSLLnPPb5h8A1wCeSnJ3kXzY9/qT9HppkXZJ1t3HLPHYnqbXFkCPXXOf5sNJi0885IR8FfpPkIcAbgJ8Dn5rHPpcC+wIfraqHAjcDb5q8UVUdWVVrq2rtttxlHruTtAg0z5HluyyZx+4kDUI/JeT2qirgWcCHqupDwHwOsV4GXFZVZ3b3j6MXJpLGlzkiaYp+SshNSQ4HDga+lmQJsO2W7rCqrgQuTXL/btHjgR9u6eNJGgnmiKQp+ikhfwbcAryse+KvBN4/z/3+JXBskvOANcC75vl4khY3c0TSFP1cO+ZK4IgJ938BfHI+O62qc4C183kMSaPDHJE0nX6OhEiSJC04S4gkSWpi1hKSZEmSTw9rGEnjxxyRNJNZS0hV/Q5Y3n0ksiTNmTkiaSb9XEX3EuC7SU6g94FAAFTVETN+hyTd2SWYI5Im6aeEXNF9bcP8PlxI0tbLHJE0RT9v0X0HQJIdqurmzW0vSZOZI5Km088F7PZL8kNgQ3f/IUn+z8AnkzQ2zBFJ0+nnLbr/ABwEXAdQVefSu4S2JPXLHJE0RV+fE1JVl05a5DWxJc2JOSJpsn5OTL00ySOB6t5idxjdIVVJ6pM5ImmKfo6EvBJ4Fb0LTl1G70JRrxrkUJLGjjkiaYp+3h1zLfCiIcwiaUyZI5KmM2MJSfJhoGZaX1WHDWQiSWPDHJE0m9mOhKwb2hSSxpU5ImlGM5aQqjpmmINIGj/miKTZbPackCTLgTcCDwK237S8qg4c4FySxog5osXkoBVrWo+gTj/vjjmW3lvp7gO8g96FqP5jgDNJGj/miKQp+ikhu1TVUcBtVXVaVb0UeMSA55I0XswRSVP082Flt3V/bkzyNHpXwtxjcCNJGkPmiKQp+ikh70xyd+B1wIeBnYDXDnQqSePGHJE0RT8fVnZid/MG4HGDHUfSODJHJE1ns+eEJNk7ySlJLuju75PkrYMfTdK4MEckTaefE1P/GTic7jXdqjoPeMEgh5I0dswRSVP0U0KWVdVZk5bdPohhJI0tc0TSFP2UkGuT7EV3/YckzwU2DnQqSePGHJE0RT/vjnkVcCTwgCSXAz8DDh7oVJLGjTkiaYp+3h1zMfCEJDsA21TVTYMfS9I4MUckTWfGEpLkr2ZYDkBVHTGgmSSNCXNE0mxmOxKyY/fn/YGHASd0958BnD7IoSSNDXNE0oxmLCFV9Q6AJCcD+246fJrkb4EvDGU6SSPNHJE0m37eHbMncOuE+7cCqwcyjaRxZY5ImqKfd8d8CjgryZfovb3uT4FjBjqVpHFjjkiaop93x/zvJP8GHNAtOqSqzh7sWJLGiTkiaTr9HAmhqtYD6xdyx0mWAOuAy6vq6Qv52JIWH3NE0mT9nBMyKK8GNjTcv6TRZ45II6xJCUmyB/A04F9a7F/S6DNHpNHX6kjIPwBvAO5otH9Jo88ckUZcX+eELKQkTweurqofJHnsLNsdChwKsOfKpZy07pwFn+WgFWsW/DElDd6W5oikxaXFkZD9gWcmuQT4LHBgkk9P3qiqjqyqtVW1dvkuS4Y9o6TFzRyRxsDQS0hVHV5Ve1TVauAFwLeqyqtpSuqbOSKNh5bvjpEkSVuxpi+SVtWpwKktZ5A02swRaXR5JESSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1MTS1gP046LzlnHQijWtx5AkjYGTrjhnII/r/1Nz55EQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDUx9BKSZFWSbyfZkOTCJK8e9gySRps5Io2HpQ32eTvwuqpan2RH4AdJvlFVP2wwi6TRZI5IY2DoR0KqamNVre9u3wRsAFYOew5Jo8sckcZDiyMhv5dkNfBQ4Mxp1h0KHAqw58qlnLTunKHOpvk5aMWa1iNoKzGXHJHAfFpMmp2YmuRuwBeB11TVjZPXV9WRVbW2qtYu32XJ8AeUtOiZI9Joa1JCkmxLLziOrarjW8wgabSZI9Loa/HumABHARuq6ohh71/S6DNHpPHQ4kjI/sD/AA5Mck739dQGc0gaXeaINAaGfqZWVX0HyLD3K2l8mCPSePATUyVJUhOWEEmS1IQlRJIkNWEJkSRJTVhCJElSE5YQSZLUhCVEkiQ1YQmRJElNWEIkSVITlhBJktSEJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRJIkNWEJkSRJTaSqWs+wWUluAn7ceo4+7Qpc23qIORileUdpVhitee9fVTu2HmKQzJGBGqV5R2lWGK1555wjSwc1yQL7cVWtbT1EP5KsG5VZYbTmHaVZYbTmTbKu9QxDYI4MyCjNO0qzwmjNuyU54ssxkiSpCUuIJElqYlRKyJGtB5iDUZoVRmveUZoVRmveUZp1S43SzzhKs8JozTtKs8JozTvnWUfixFRJkjR+RuVIiCRJGjOWEEmS1MSiKyFJPp7k6iQXTFj2/iQ/SnJeki8l2bnljJtMN+uEdX+dpJLs2mK26cw0b5K/TPLjJBcmeV+r+Saa4d/BmiTfT3JOknVJHt5yxk2SrEry7SQbur/DV3fL75nkG0l+0v15j9azwqzzLsrn2ZYwRwZjlDIEzJFBWrAcqapF9QU8GtgXuGDCsicBS7vb7wXe23rOmWbtlq8CTgJ+Duzaes7N/N0+DvgmcJfu/r1azznLrCcDT+luPxU4tfWc3Sy7A/t2t3cELgIeBLwPeFO3/E2L6N/tTPMuyufZAv77WZQ/3yjlyChlyCzzmiODnXdOz7NFdySkqk4Hfjlp2clVdXt39/vAHkMfbBrTzdr5IPAGYFGd9TvDvH8BvKeqbum2uXrog01jhlkL2Km7fXfgiqEONYOq2lhV67vbNwEbgJXAs4Bjus2OAZ7dZsI7m2nexfo82xLmyGCMUoaAOTJIC5Uji66E9OGlwL+1HmImSZ4JXF5V57aepU97AwckOTPJaUke1nqgWbwGeH+SS4EPAIc3nmeKJKuBhwJnArtV1UboPWGBe7WbbHqT5p1oUT/PFsCi/vlGLEdGKUPAHFlw88mRkSohSd4C3A4c23qW6SRZBrwFeFvrWeZgKXAP4BHA64HPJ0nbkWb0F8Brq2oV8FrgqMbz3EmSuwFfBF5TVTe2nmdzZpp3sT/P5mux/3wjmCOjlCFgjiyo+ebIyJSQJC8Gng68qLoXmxahvYD7AOcmuYTeYaj1Sf6g6VSzuww4vnrOAu6gd8GkxejFwPHd7S8Ai+KEMoAk29J7Ih5bVZtmvCrJ7t363YFFc5h6hnlH5Xm2xUbk5xu1HBmlDAFzZMEsRI6MRAlJ8mTgjcAzq+o3reeZSVWdX1X3qqrVVbWa3pNz36q6svFos/kycCBAkr2B7Vi8V2y8AnhMd/tA4CcNZ/m97re+o4ANVXXEhFUn0As8uj+/MuzZpjPTvKPyPNtSo/LzjWCOjFKGgDmyIBYsR1qfYTv5C/gMsBG4jd6T72XAT4FLgXO6r4+1nnOmWSetv4RFclb7LH+32wGfBi4A1gMHtp5zllkfBfwAOJfea49/3HrObtZH0TvZ7bwJ/0afCuwCnEIv5E4B7tl61s3MuyifZwv472dR/nyjlCOjlCGzzGuODHbeOT3P/Nh2SZLUxEi8HCNJksaPJUSSJDVhCZEkSU1YQiRJUhOWEEmS1IQlRCQ5Ncna1nNIGl3miLaEJUQDl2RJ6xkkjTZzZDxZQrYSSVYn+VGSY5Kcl+S47hoVk7f7aJJ1SS5M8o5u2eOTfGnCNk9Mcnx3+0lJvpdkfZIvdNcRIMklSd6W5DvA8ybt43lJLkhybpLTu2UvSfKRCducmOSx3e1fJ3lvkh8k+WaSh3e/dV3cXehL0hCYI1polpCty/2BI6tqH+BG4H9Ns81bqmotsA/wmCT7AN8CHphkebfNIcAnkuwKvBV4QlXtC6wD/mrCY/22qh5VVZ+dtI+3AQdV1UOAfp78OwCnVtUfAzcB7wSeCPwp8Hd9fL+khWOOaMFYQrYul1bVd7vbn6b3sbuTPT/JeuBs4MHAg6r3sbqfAg5OsjOwH73LMz8CeBDw3STn0Luuwb0nPNbnZpjju8DRSV4O9HOI9Vbg37vb5wOnVdVt3e3VfXy/pIVjjmjBLG09gIZq8mf03+l+kvsAfw08rKp+leRoYPtu9SeArwK/Bb5QVbd3FzD6RlW9cIb93TztEFWvTPInwNOAc5KsoXfJ54mlePsJt2+r/7q+wB3ALd3j3JHEf8PScJkjWjAeCdm67Jlkv+72C4HvTFq/E70n/A1JdgOesmlFVV1B7+qTbwWO7hZ/H9g/yX0BkizrrqI5qyR7VdWZVfU2elfbXEXvIl1rkmyTZBWL6PLaku7EHNGCsf1tXTYAL07yT/SuyPjRiSur6twkZwMXAhfTO9w50bHA8qr6Ybf9NUleAnwmyV26bd4KXLSZOd6f5H5A6F0V8txu+c/oHRrddDVOSYuPOaIF41V0txJJVgMnVtUfzeMxPgKcXVVHLdRckkaHOaKF5pEQ9SXJD+gdYn1d61kkjSZzRJN5JESSJDXhiamSJKkJS4gkSWrCEiJJkpqwhEiSpCYsIZIkqYn/D6CdL1VRJng2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiEAAAEWCAYAAACwgEcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHpVJREFUeJzt3XmYZAV57/Hvb3r2YQYGBEVEMSgoJopkNOIuLlHjkuTGxFy51y0Sc32iMYvRxJiYxxujJiYmea6Ge4kQ5apRMSJmwWBE9LoEkVXiRkCWAYYdBpj1vX/UGdNML1Pdc06frprv53n66apT1e95q6fOb94+dapOqgpJkqSFtqTvBiRJ0r7JIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIUSSJPXCIWQflOSuJD8yy+1XJnnWQvYkaTyZN5qNQ8g+qKr2q6orAJKcmuQdC7HeJK9I8qWFWJekxaGvvNFocAiRJEm9cAgZE0lemeQzk65/L8nfTbp+dZJjm8uV5GFJTgJeBryp2WX6mUklj01ycZLbk3wsycpJtV7T1L8lyZlJHtgsP6KpvXTSfb+Q5JeSPBL4AHB8s67bZnkclye5M8kVSX55t9tfnOTCJHck+X6S5zbL909ySpKNSa5N8o4kE3vxK5U0g4XMm93We2SSzye5OclNSU5PcsCk2w9PckaSTc19/mrSba9qsuXWJP+c5CEt/ko0Tw4h4+Nc4ClJliQ5FFgGPAmgeT12P+DiyT9QVScDpwPvbnaZvnDSzT8PPBd4KPBo4BVNrROAdza3HwpcBXx0T81V1eXAa4GvNOs6YIa73gi8AFgHvBL4syTHNet+PPC3wG8BBwBPBa5sfu40YDvwMOCxwHOAX9pTX5LmZUHyZhphkD8PBB4JHA78QbPeCeAsBpl0BHAYTTYl+Wngd4CfBQ4GzgM+Ms/HrhYt3fNdNAqq6ookdwLHAkcB/8zgr4tHAMcD51XVzjmU/Iuqug6g+Yvl2Gb5y4C/qaoLmtveAtya5IiWHsdnJ109N8nZwFOAC4BXN+v+XHP7tU0P9weeBxxQVfcAm5P8GXAS8Ndt9CXpPy1g3uy+3u8B32uubkryXuD3m+uPZzCc/FZVbW+W7ToG7ZeBdzZ/DJHkj4DfSfKQqrpqDn2qZQ4h4+Vc4OkM9gacC9wGPI1BKJw7x1rXT7p8N4ONm+b7BbtuqKq7ktzM4K+Oa+fV9SRJnscgVI5isKduNXBJc/PhwD9M82MPYfCX2MYku5YtAa7e234kzWgh8uY+khwC/AWDP0zWMtjOb21uPhy4atIAMtlDgPcl+dPJ5RjklkNIj3w5ZrzsCoWnNJfPZRAKT2PmUJjraZSvY7BBA5BkDXAQgwFkc7N49aT7P2DYdSVZAXwS+BPg/s1LNv/AICxgMFQcOc2PXg1sAe5XVQc0X+uq6lFDPypJc7UQebO7dzY1Hl1V64ATuW8+PHjyMWmTXA388qR8OKCqVlXV/9vLfrSXHELGy7nAM4BVVXUNg9c9n8tgSPjmDD9zAzDje/in8X+BVyY5thka/gj4WlVdWVWbGAwjJyaZSPIq7js03AA8KMnyGWovB1YAm4DtzV6R50y6/ZRm3c9sXos+LMkjqmojcDbwp0nWNbcdmeRpc3hckuZmIfJmd2uBu4DbkhzG4PiwXb4ObAT+OMmaJCuTPKm57QPAW5I8Cn54IPtL9qIPtcQhZIxU1XcYbKDnNdfvAK4AvlxVO2b4sVOAY5LcluTvh1jHOcDvMdhjsZHBkPHSSXd5DYNguBl4FDD5L43PA5cB1ye5aZradwKvB/6OwS7W/wqcOen2r9McrArcziAEd+2V+e8MhphvNT/7CQYHzkrqwELkzTTeDhzHYPv/LHDGpH52AC9k8PLQD4BrgF9obvsU8C7go0nuAC5lcByZepaqvd07JkmSNHfuCZEkSb1wCJEkSb1wCJEkSb1wCJEkSb0YiQ8rW750da1atn/fbQxvx1w+KHBYHR1AvLOLXoGl3Ty1dqzqqO7K7PlOczSxdlvrNQGOXHlz6zWvvmYHt9yys/1fwiKydNWaWr72wNbrLpnuo7Ha0MEmn53d5Eg6iqeu3jZRy7p5qu/sIJ6yrpsn2MNWTXmD4l6bT46MxBCyatn+HP+wV7dfePtM7yLbO9l8T/tFd3TTa22+u5O6HHJQJ2XvetTBndS99ej2N4X1z9zYek2Ajz3yQ63XfP7z2w+kxWb52gM56iVvbL3u6k3dDPLpYJNfek83vU7c21E+LelmWLjnkGWd1L37kPZfXFj+k5tarwnw6R/7YOs155MjvhwjSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ64RAiSZJ60dkQkuRvktyY5NJJyw5M8rkk322+r+9q/ZJGnzkijbcu94ScCjx3t2VvBs6pqocD5zTXJWkmp2KOSGOrsyGkqr4I3LLb4hcDpzWXTwN+uqv1Sxp95og03pYu8PruX1UbAapqY5JDZrpjkpOAkwBWrtif7fuvXKAW997EivZ/rdm2o/WaAFnSzRy6Y+3qTupuX9lNv3cdvbX1mn965Fmt1wQ4dOl+rddclltbr9mheeXIsrXrqYm03swNG7p5Tu53dfu9bl810XpNgFU3dfNfyfZVnZRl5a3VSd3bj9nees33H31m6zVh8eTIoj0wtapOrqoNVbVh2dI1fbcjaQRNzpGlq8wRabFZ6CHkhiSHAjTfb1zg9UsafeaINCYWegg5E3h5c/nlwKcXeP2SRp85Io2JLt+i+xHgK8DRSa5J8mrgj4FnJ/ku8OzmuiRNyxyRxltnB6ZW1S/OcNMzu1qnpPFijkjjbdEemCpJksabQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSeqFQ4gkSerF0r4bGEYtCdtXt9/q0nt2tF4TYOfq5a3XzJZues2yiU7qMpFOym768W7qPu/Rl7Zec3m6+TfbVu3XLar1movNzgnYckD7dVdf381zcuv+7decuLf9mgBb143O7wDgnvt30++zHntJ6zVXZlvrNWHx5Ih7QiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi96GUKSvDHJZUkuTfKRJCv76EPS6DJHpNG34ENIksOA1wMbqupHgQngpQvdh6TRZY5I46Gvl2OWAquSLAVWA9f11Iek0WWOSCNu6UKvsKquTfInwA+Ae4Czq+rsWX9mArbvN9F6LztXdDODLbt9W+s1u5oWl9zRfq8Amx63tpO6ax9xSzd1l97bes17a1nrNQe6+TcbJfPNka3rd3bQTTdb50T7T0lIBzWBFbdVJ3WXbO2kLBMvuLmTug9ceVvrNZeki+csbKsdrdecz7Ogj5dj1gMvBh4KPBBYk+TEae53UpLzk5y/fcvmhW5T0iI2nxzZsdkckRabPl6OeRbwH1W1qaq2AWcAT9z9TlV1clVtqKoNS1esWfAmJS1qc86RiTXmiLTY9DGE/AB4QpLVSQI8E7i8hz4kjS5zRBoDCz6EVNXXgE8AFwCXND2cvNB9SBpd5og0Hhb8wFSAqvp94Pf7WLek8WCOSKPPT0yVJEm9cAiRJEm92OPLMUl+dprFtwOXVNWN7bckadyYI5KmM8wxIa8Gjgf+tbn+dOCrwFFJ/rCqPtRRb5LGhzkiaYphhpCdwCOr6gaAJPcH3g/8BPBFwPCQtCfmiKQphjkm5IhdwdG4ETiqqm7Bz4+WNBxzRNIUw+wJOS/JWcDHm+v/BfhikjVA+x+UL2kcmSOSphhmCHkdg8B4EoPTH/0t8MmqKuAZHfYmaXyYI5Km2OMQ0oTEJ5ovSZozc0TSdPZ4TEiSn03y3SS3J7kjyZ1J7liI5iSNB3NE0nSGeTnm3cALq8qTQ0maL3NE0hTDvDvmBoND0l4yRyRNMcyekPOTfAz4e2DLroVVdUZnXUkaN+aIpCmGGULWAXcDz5m0rADDQ9KwzBFJUwzz7phXLkQjksaXOSJpOjMOIUneVFXvTvKXDP5iuY+qen2nnUkaeeaIpNnMtidk10Fk5y9EI7OpiXDvARMdFG6/JEAOGuZVrrlZsq2bZrevXNNJ3c2HpZO6W27tpt/L1h3aes2HrtjUek2Au1Z8t/WaO7raGBZRjmQnLLtzmGPx52brATtbrwmwZFv721C6aZW7fmxrJ3UnblzeSd3tN67rpO73Dzy49ZoPXv6A1msCHLPs+63X3DmPHJnxf8uq+kxz8byqumK+TUnad5kjkmYzzJ/spyY5DPg3Bme7PK+qLum2LUljxhyRNMUwB6Y+Ncly4HHA04HPJtmvqg7sujlJ48EckTSdPQ4hSZ4MPKX5OgA4Cziv474kjRFzRNJ0hnk55lwGB5W9E/iHqurmCCRJ48wckTTFMEPIQQxOv/1U4PVJdgJfqarf67QzSePEHJE0xTDHhNyW5ArgcOBBwBOBZV03Jml8mCOSpjPMMSHfB74NfAn4APBKd6VKmgtzRNJ0hnk55uFV1dFH3EjaR5gjkqYY5uMDH5jkU0luTHJDkk8meVDnnUkaJ+aIpCmGGUI+CJwJPBA4DPhMs0yShmWOSJpimCHk4Kr6YFVtb75OBdr/gHxJ48wckTTFMEPITUlOTDLRfJ0I3Nx1Y5LGijkiaYphhpBXAT8PXA9sBH6uWSZJwzJHJE0xzOeE/AB40QL0ImlMmSOSpjPM54QcDLwGOGLy/avKv2IkDcUckTSdYT4n5NMMTjT1L8CObtuRNKbMEUlTDDOErK6q3+68E0njzByRNMUwB6aeleT5ba40yQFJPpHk35NcnuT4NutLWnTMEUlTzLgnJMmdQAEBfifJFmBbc72qat1erPd9wD9V1c8lWQ6s3otakhYpc0TSbGYcQqpqbRcrTLKOwem8X9GsZyvgiaykMWSOSJrNMO+OeRJwYVVtbj5g6Djgz5u33M3HjwCbgA8meQzwDeANVbV5t/WeBJwEsHzNepZsr3mubma1JK3XBDrpdefSbnqtiU7KsvyObureu3mYw5jm7uZ72v8j+t7q5kz1O2j/+dW1xZAjE/fbn60Pu2fej2EmK769qvWaANnefs0dq7p57izZtLyTukvv6ib3sr2bbfOOI1e2XnMi3Zz3sYuq83l2DXNMyPuBu5sN/U3AVcCH5rGuXZYyCKD3V9Vjgc3Am3e/U1WdXFUbqmrD0hVr9mJ1khaB3nNkYq05Ii02wwwh26uqgBcD76uq9wF7s4v1GuCaqvpac/0TDMJE0vgyRyRNMcwQcmeStwAnAp9NMgHMe19WVV0PXJ3k6GbRM4FvzbeepJFgjkiaYpgh5BeALcCrmw3/MOA9e7neXwVOT3IxcCzwR3tZT9LiZo5ImmKYc8dcD7x30vUfAH+7NyutqguBDXtTQ9LoMEckTWeYPSGSJEmtcwiRJEm9mHUISTKR5MML1Yyk8WOOSJrJrENIVe0ADm4+ElmS5swckTSTYT5+8krgy0nOZPCBQABU1Xtn/AlJuq8rMUck7WaYIeS65msJe/fhQpL2XeaIpCmGeYvu2wGSrNn9vAySNAxzRNJ09vjumCTHJ/kWcHlz/TFJ/lfnnUkaG+aIpOkM8xbdPwd+ErgZoKouYnAKbUkaljkiaYqhPiekqq7ebdGODnqRNMbMEUm7G+bA1KuTPBGo5i12r6fZpSpJQzJHJE0xzJ6Q1wKvY3DCqWsYnCjqdV02JWnsmCOSphjm3TE3AS9bgF4kjSlzRNJ0ZhxCkvwlUDPdXlWv76QjSWPDHJE0m9n2hJy/YF1IGlfmiKQZzTiEVNVpC9mIpPFjjkiazR6PCUlyMPDbwDHAyl3Lq+qEDvu6bw87YdnmGffoztv2lXu+z7yk/ZJLdrT/+AGW39xN3ewc6t3fc6+7rZu6W7YN80axublrR1dPsNGzGHKEHWHnnctaL5vtrZcEYPt+7W+bNdF6SQBW3NpB6HXo3vXdvDv8qlvXt17zO+se0HpNgGetvqL1mlVzf84Ok+inM3gr3UOBtzM4EdW/zXlNkvZl5oikKYYZQg6qqlOAbVV1blW9CnhCx31JGi/miKQphtkHva35vjHJTzE4E+aDumtJ0hgyRyRNMcwQ8o4k+wO/AfwlsA54Y6ddSRo35oikKYb5sLKzmou3A8/oth1J48gckTSdPR4TkuSoJOckubS5/ugkb+2+NUnjwhyRNJ1hDkz938BbaF7TraqLgZd22ZSksWOOSJpimCFkdVV9fbdlHb0zXtKYMkckTTHMEHJTkiNpzv+Q5OeAjZ12JWncmCOSphjm3TGvA04GHpHkWuA/gBM77UrSuDFHJE0xzLtjrgCelWQNsKSq7uy+LUnjxByRNJ0Zh5Akvz7DcgCq6r0d9SRpTJgjkmYz256Qtc33o4HHAWc2118IfLHLpiSNDXNE0oxmHEKq6u0ASc4Gjtu1+zTJHwAfX5DuJI00c0TSbIZ5d8yDga2Trm8FjuikG0njyhyRNMUw7475EPD1JJ9i8Pa6nwFO67QrSePGHJE0xTDvjvmfSf4ReEqz6JVV9c1u25I0TswRSdMZZk8IVXUBcEGbK04yAZwPXFtVL2iztqTFxxyRtLthjgnpyhuAy3tcv6TRZ45II6yXISTJg4CfAv5PH+uXNPrMEWn09bUn5M+BNwE7e1q/pNFnjkgjbqhjQtqU5AXAjVX1jSRPn+V+JwEnAaxYdQCpar2Xia17vs/86rbfa3U0LtZEuinckezopu7q5dtar7mso2a7+B+3/Wdst+aTIxPr15Ot7W9Iy+5qvSQAq69vv+bWdd1s7ytu6+YZtGV9R/nUUdkdO9t/fm3Z2c1/0xMd1JzPr7WPPSFPAl6U5Ergo8AJST68+52q6uSq2lBVG5auWLPQPUpa3OacIxP7mSPSYrPgQ0hVvaWqHlRVRwAvBT5fVZ5NU9LQzBFpPPT57hhJkrQPW/BjQiarqi8AX+izB0mjzRyRRpd7QiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi8cQiRJUi+W9t3AMFKQ7e3XXUK1XxSoDka7Jdu66ZWO6i7Z3s18mx3ppO7mLctbr7mjiycCsLXa/zerDmouOilqxc7Wy95+bAfhBLCt/efPmiu7ifxtHW2X9x7UUT7d3c22uf/qe1qvedCyza3XBLi3g1/tfLYu94RIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReOIRIkqReLPgQkuTwJP+a5PIklyV5w0L3IGm0mSPSeFjawzq3A79RVRckWQt8I8nnqupbPfQiaTSZI9IYWPA9IVW1saouaC7fCVwOHLbQfUgaXeaINB762BPyQ0mOAB4LfG2a204CTgJYObEf+51/VfsNrFjefk2gbr+j9Zo779rcek0AJiY6Kbtiy5ZO6h74mEd2UveGJ65vveYpRz+j9ZoAO09I6zVv3nFe6zUXytA5wmqOeu3XW1//7Sc+ofWaAGuvbn8bmth8d+s1AZbc3k1dbrmtk7I7juxmXr394Ye0XvOUDe3XBLj96atar7lpxzlz/pneDkxNsh/wSeDXqmrK/9pVdXJVbaiqDcuXtP/LkjT65pIjy1ix8A1KmlUvQ0iSZQyC4/SqOqOPHiSNNnNEGn19vDsmwCnA5VX13oVev6TRZ45I46GPPSFPAv4bcEKSC5uv5/fQh6TRZY5IY2DBD0ytqi8B7R9ZJ2mfYY5I48FPTJUkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1wCJEkSb1IVfXdwx4luRP4dt99DOl+wE19NzEHo9TvKPUKo9Xv0VW1tu8mumSOdGqU+h2lXmG0+p1zjiztqpOWfbuqNvTdxDCSnD8qvcJo9TtKvcJo9Zvk/L57WADmSEdGqd9R6hVGq9/55Igvx0iSpF44hEiSpF6MyhByct8NzMEo9Qqj1e8o9Qqj1e8o9Tpfo/QYR6lXGK1+R6lXGK1+59zrSByYKkmSxs+o7AmRJEljxiFEkiT1YtENIUn+JsmNSS6dtOw9Sf49ycVJPpXkgD573GW6Xifd9ptJKsn9+uhtOjP1m+RXk3w7yWVJ3t1Xf5PN8Dw4NslXk1yY5Pwkj++zx12SHJ7kX5Nc3vwO39AsPzDJ55J8t/m+vu9eYdZ+F+V2Nh/mSDdGKUPAHOlSazlSVYvqC3gqcBxw6aRlzwGWNpffBbyr7z5n6rVZfjjwz8BVwP367nMPv9tnAP8CrGiuH9J3n7P0ejbwvOby84Ev9N1n08uhwHHN5bXAd4BjgHcDb26Wv3kRPW9n6ndRbmctPn8W5eMbpRwZpQyZpV9zpNt+57SdLbo9IVX1ReCW3ZadXVXbm6tfBR604I1NY7peG38GvAlYVEf9ztDvrwB/XFVbmvvcuOCNTWOGXgtY11zeH7huQZuaQVVtrKoLmst3ApcDhwEvBk5r7nYa8NP9dHhfM/W7WLez+TBHujFKGQLmSJfaypFFN4QM4VXAP/bdxEySvAi4tqou6ruXIR0FPCXJ15Kcm+RxfTc0i18D3pPkauBPgLf03M8USY4AHgt8Dbh/VW2EwQYLHNJfZ9Pbrd/JFvV21oJF/fhGLEdGKUPAHGnd3uTISA0hSX4X2A6c3ncv00myGvhd4G199zIHS4H1wBOA3wL+Lkn6bWlGvwK8saoOB94InNJzP/eRZD/gk8CvVdUdffezJzP1u9i3s7212B/fCObIKGUImCOt2tscGZkhJMnLgRcAL6vmxaZF6EjgocBFSa5ksBvqgiQP6LWr2V0DnFEDXwd2Mjhh0mL0cuCM5vLHgUVxQBlAkmUMNsTTq2pXjzckObS5/VBg0eymnqHfUdnO5m1EHt+o5cgoZQiYI61pI0dGYghJ8lzgt4EXVdXdffczk6q6pKoOqaojquoIBhvncVV1fc+tzebvgRMAkhwFLGfxnrHxOuBpzeUTgO/22MsPNX/1nQJcXlXvnXTTmQwCj+b7pxe6t+nM1O+obGfzNSqPbwRzZJQyBMyRVrSWI30fYbv7F/ARYCOwjcHG92rge8DVwIXN1wf67nOmXne7/UoWyVHts/xulwMfBi4FLgBO6LvPWXp9MvAN4CIGrz3+eN99Nr0+mcHBbhdPeo4+HzgIOIdByJ0DHNh3r3vod1FuZy0+fxbl4xulHBmlDJmlX3Ok237ntJ35se2SJKkXI/FyjCRJGj8OIZIkqRcOIZIkqRcOIZIkqRcOIZIkqRcOISLJF5Js6LsPSaPLHNF8OISoc0km+u5B0mgzR8aTQ8g+IskRSf49yWlJLk7yieYcFbvf7/1Jzk9yWZK3N8uemeRTk+7z7CRnNJefk+QrSS5I8vHmPAIkuTLJ25J8CXjJbut4SZJLk1yU5IvNslck+atJ9zkrydOby3cleVeSbyT5lySPb/7quqI50ZekBWCOqG0OIfuWo4GTq+rRwB3A/5jmPr9bVRuARwNPS/Jo4PPAI5Mc3NznlcAHk9wPeCvwrKo6Djgf+PVJte6tqidX1Ud3W8fbgJ+sqscAw2z8a4AvVNWPA3cC7wCeDfwM8IdD/Lyk9pgjao1DyL7l6qr6cnP5www+dnd3P5/kAuCbwKOAY2rwsbofAk5McgBwPIPTMz8BOAb4cpILGZzX4CGTan1shj6+DJya5DXAMLtYtwL/1Fy+BDi3qrY1l48Y4ucltcccUWuW9t2AFtTun9F/n+tJHgr8JvC4qro1yanAyubmDwKfAe4FPl5V25sTGH2uqn5xhvVtnraJqtcm+Qngp4ALkxzL4JTPk4filZMub6v/PL/ATmBLU2dnEp/D0sIyR9Qa94TsWx6c5Pjm8i8CX9rt9nUMNvjbk9wfeN6uG6rqOgZnn3wrcGqz+KvAk5I8DCDJ6uYsmrNKcmRVfa2q3sbgbJuHMzhJ17FJliQ5nEV0em1J92GOqDVOf/uWy4GXJ/lrBmdkfP/kG6vqoiTfBC4DrmCwu3Oy04GDq+pbzf03JXkF8JEkK5r7vBX4zh76eE+ShwNhcFbIi5rl/8Fg1+ius3FKWnzMEbXGs+juI5IcAZxVVT+6FzX+CvhmVZ3SVl+SRoc5ora5J0RDSfINBrtYf6PvXiSNJnNEu3NPiCRJ6oUHpkqSpF44hEiSpF44hEiSpF44hEiSpF44hEiSpF78fwUuXTOvibDBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(policy.argmax(-1))\n",
    "plot(v)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
